2024-11-16 05:55:54,375 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 05:55:54,388 main DEBUG Took 0.011576 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 05:55:54,389 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 05:55:54,389 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 05:55:54,390 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 05:55:54,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,400 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 05:55:54,415 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,417 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,418 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,420 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,421 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,421 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,423 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,424 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,425 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,426 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,427 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,428 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,429 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:55:54,430 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,431 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 05:55:54,433 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:55:54,434 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 05:55:54,437 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 05:55:54,438 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 05:55:54,440 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 05:55:54,440 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 05:55:54,449 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 05:55:54,452 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 05:55:54,454 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 05:55:54,454 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 05:55:54,455 main DEBUG createAppenders(={Console}) 2024-11-16 05:55:54,456 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 05:55:54,456 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 05:55:54,456 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 05:55:54,457 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 05:55:54,458 main DEBUG OutputStream closed 2024-11-16 05:55:54,458 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 05:55:54,458 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 05:55:54,459 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 05:55:54,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 05:55:54,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 05:55:54,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 05:55:54,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 05:55:54,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 05:55:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 05:55:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 05:55:54,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 05:55:54,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 05:55:54,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 05:55:54,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 05:55:54,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 05:55:54,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 05:55:54,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 05:55:54,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 05:55:54,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 05:55:54,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 05:55:54,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 05:55:54,558 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 05:55:54,558 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 05:55:54,559 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 05:55:54,559 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T05:55:54,777 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e 2024-11-16 05:55:54,779 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 05:55:54,779 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T05:55:54,787 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-16T05:55:54,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T05:55:54,824 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663, deleteOnExit=true 2024-11-16T05:55:54,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T05:55:54,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/test.cache.data in system properties and HBase conf 2024-11-16T05:55:54,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T05:55:54,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.log.dir in system properties and HBase conf 2024-11-16T05:55:54,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T05:55:54,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T05:55:54,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T05:55:54,937 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T05:55:55,027 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:55,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:55:55,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:55:55,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T05:55:55,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:55:55,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T05:55:55,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T05:55:55,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:55:55,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:55:55,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T05:55:55,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/nfs.dump.dir in system properties and HBase conf 2024-11-16T05:55:55,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/java.io.tmpdir in system properties and HBase conf 2024-11-16T05:55:55,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:55:55,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T05:55:55,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T05:55:55,764 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T05:55:55,833 INFO [Time-limited test {}] log.Log(170): Logging initialized @2119ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T05:55:55,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:55:55,965 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:55:55,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:55:55,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:55:55,987 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:55:55,998 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:55:56,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:55:56,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:55:56,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/java.io.tmpdir/jetty-localhost-34751-hadoop-hdfs-3_4_1-tests_jar-_-any-10281326168303826246/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:55:56,175 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34751} 2024-11-16T05:55:56,175 INFO [Time-limited test {}] server.Server(415): Started @2462ms 2024-11-16T05:55:56,493 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:55:56,501 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:55:56,502 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:55:56,502 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:55:56,503 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T05:55:56,503 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:55:56,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:55:56,603 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/java.io.tmpdir/jetty-localhost-34551-hadoop-hdfs-3_4_1-tests_jar-_-any-16146517420874458946/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:55:56,604 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34551} 2024-11-16T05:55:56,604 INFO [Time-limited test {}] server.Server(415): Started @2891ms 2024-11-16T05:55:56,653 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:55:57,098 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data1/current/BP-1783175189-172.17.0.2-1731736555553/current, will proceed with Du for space computation calculation, 2024-11-16T05:55:57,098 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data2/current/BP-1783175189-172.17.0.2-1731736555553/current, will proceed with Du for space computation calculation, 2024-11-16T05:55:57,137 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:55:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x566bfa8278bbe498 with lease ID 0x76e380e1c74993b5: Processing first storage report for DS-a1076f56-66f6-48f8-8aac-e785a709d653 from datanode DatanodeRegistration(127.0.0.1:41419, datanodeUuid=b3959656-3ba6-40ef-9099-78748206e8cb, infoPort=44519, infoSecurePort=0, ipcPort=35371, storageInfo=lv=-57;cid=testClusterID;nsid=872440879;c=1731736555553) 2024-11-16T05:55:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x566bfa8278bbe498 with lease ID 0x76e380e1c74993b5: from storage DS-a1076f56-66f6-48f8-8aac-e785a709d653 node DatanodeRegistration(127.0.0.1:41419, datanodeUuid=b3959656-3ba6-40ef-9099-78748206e8cb, infoPort=44519, infoSecurePort=0, ipcPort=35371, storageInfo=lv=-57;cid=testClusterID;nsid=872440879;c=1731736555553), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-16T05:55:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x566bfa8278bbe498 with lease ID 0x76e380e1c74993b5: Processing first storage report for DS-0e4cc6e1-8ebf-460e-b6e1-8f98d3db3e09 from datanode DatanodeRegistration(127.0.0.1:41419, datanodeUuid=b3959656-3ba6-40ef-9099-78748206e8cb, infoPort=44519, infoSecurePort=0, ipcPort=35371, storageInfo=lv=-57;cid=testClusterID;nsid=872440879;c=1731736555553) 2024-11-16T05:55:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x566bfa8278bbe498 with lease ID 0x76e380e1c74993b5: from storage DS-0e4cc6e1-8ebf-460e-b6e1-8f98d3db3e09 node DatanodeRegistration(127.0.0.1:41419, datanodeUuid=b3959656-3ba6-40ef-9099-78748206e8cb, infoPort=44519, infoSecurePort=0, ipcPort=35371, storageInfo=lv=-57;cid=testClusterID;nsid=872440879;c=1731736555553), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:55:57,253 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e 2024-11-16T05:55:57,318 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/zookeeper_0, clientPort=59104, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:55:57,328 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59104 2024-11-16T05:55:57,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:57,346 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:55:57,981 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 with version=8 2024-11-16T05:55:57,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/hbase-staging 2024-11-16T05:55:58,060 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T05:55:58,256 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:58,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,269 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:58,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:58,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:55:58,460 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T05:55:58,468 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T05:55:58,471 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:58,495 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 95117 (auto-detected) 2024-11-16T05:55:58,496 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T05:55:58,514 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40477 2024-11-16T05:55:58,535 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40477 connecting to ZooKeeper ensemble=127.0.0.1:59104 2024-11-16T05:55:58,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404770x0, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:58,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40477-0x100471a33620000 connected 2024-11-16T05:55:58,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:58,596 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:58,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:58,608 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79, hbase.cluster.distributed=false 2024-11-16T05:55:58,628 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:58,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40477 2024-11-16T05:55:58,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40477 2024-11-16T05:55:58,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40477 2024-11-16T05:55:58,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40477 2024-11-16T05:55:58,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40477 2024-11-16T05:55:58,732 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:58,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,734 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:58,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:58,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:58,736 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:58,739 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:58,740 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45735 2024-11-16T05:55:58,742 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45735 connecting to ZooKeeper ensemble=127.0.0.1:59104 2024-11-16T05:55:58,743 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:58,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:58,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457350x0, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:58,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:457350x0, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:58,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45735-0x100471a33620001 connected 2024-11-16T05:55:58,758 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:58,765 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:58,767 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:55:58,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:58,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45735 2024-11-16T05:55:58,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45735 2024-11-16T05:55:58,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45735 2024-11-16T05:55:58,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45735 2024-11-16T05:55:58,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45735 2024-11-16T05:55:58,792 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:40477 2024-11-16T05:55:58,793 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,40477,1731736558105 2024-11-16T05:55:58,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:58,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:58,801 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,40477,1731736558105 2024-11-16T05:55:58,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:58,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:58,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:58,821 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:55:58,822 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,40477,1731736558105 from backup master directory 2024-11-16T05:55:58,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:58,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,40477,1731736558105 2024-11-16T05:55:58,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:58,826 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:58,826 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,40477,1731736558105 2024-11-16T05:55:58,828 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T05:55:58,829 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T05:55:58,880 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/hbase.id] with ID: 56d35825-0086-4de4-ab79-2eff0746a20d 2024-11-16T05:55:58,880 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/.tmp/hbase.id 2024-11-16T05:55:58,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:55:59,295 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/.tmp/hbase.id]:[hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/hbase.id] 2024-11-16T05:55:59,348 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:55:59,353 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:55:59,371 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T05:55:59,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:59,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:59,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:55:59,815 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:55:59,817 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:55:59,830 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:55:59,834 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:59,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:56:00,287 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store 2024-11-16T05:56:00,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:56:00,717 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T05:56:00,720 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:00,721 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:56:00,722 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:56:00,722 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:56:00,723 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:56:00,724 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:56:00,724 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:56:00,726 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736560721Disabling compacts and flushes for region at 1731736560721Disabling writes for close at 1731736560724 (+3 ms)Writing region close event to WAL at 1731736560724Closed at 1731736560724 2024-11-16T05:56:00,728 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/.initializing 2024-11-16T05:56:00,729 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/WALs/3456ee6a3164,40477,1731736558105 2024-11-16T05:56:00,737 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:56:00,751 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C40477%2C1731736558105, suffix=, logDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/WALs/3456ee6a3164,40477,1731736558105, archiveDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/oldWALs, maxLogs=10 2024-11-16T05:56:00,771 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/WALs/3456ee6a3164,40477,1731736558105/3456ee6a3164%2C40477%2C1731736558105.1731736560755, exclude list is [], retry=0 2024-11-16T05:56:00,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41419,DS-a1076f56-66f6-48f8-8aac-e785a709d653,DISK] 2024-11-16T05:56:00,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T05:56:00,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/WALs/3456ee6a3164,40477,1731736558105/3456ee6a3164%2C40477%2C1731736558105.1731736560755 2024-11-16T05:56:00,822 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44519:44519)] 2024-11-16T05:56:00,822 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:56:00,823 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:00,826 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,827 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:56:00,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:00,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:00,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,890 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:56:00,890 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:00,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:00,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,894 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:56:00,894 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:00,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:00,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:56:00,898 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:00,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:00,899 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,903 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,904 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,909 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,910 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,914 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:56:00,918 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:56:00,923 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:56:00,924 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72266671, jitterRate=0.0768573135137558}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:56:00,930 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731736560839Initializing all the Stores at 1731736560841 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736560841Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736560842 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736560842Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736560842Cleaning up temporary data from old regions at 1731736560910 (+68 ms)Region opened successfully at 1731736560930 (+20 ms) 2024-11-16T05:56:00,932 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:56:00,962 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fa0147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:56:00,992 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:56:01,003 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T05:56:01,003 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T05:56:01,005 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:56:01,007 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:56:01,011 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:56:01,011 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:56:01,039 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:56:01,049 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:56:01,051 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:56:01,053 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:56:01,054 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:56:01,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:56:01,058 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:56:01,062 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:56:01,065 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:56:01,067 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:56:01,070 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:56:01,086 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:56:01,089 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:56:01,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:56:01,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:56:01,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,098 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,40477,1731736558105, sessionid=0x100471a33620000, setting cluster-up flag (Was=false) 2024-11-16T05:56:01,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,122 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:56:01,124 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,40477,1731736558105 2024-11-16T05:56:01,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:01,136 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:56:01,138 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,40477,1731736558105 2024-11-16T05:56:01,144 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:56:01,185 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(746): ClusterId : 56d35825-0086-4de4-ab79-2eff0746a20d 2024-11-16T05:56:01,187 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:56:01,192 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:56:01,192 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:56:01,195 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:56:01,196 DEBUG [RS:0;3456ee6a3164:45735 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1283c177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:56:01,209 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:45735 2024-11-16T05:56:01,212 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:56:01,212 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:56:01,212 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:56:01,214 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40477,1731736558105 with port=45735, startcode=1731736558702 2024-11-16T05:56:01,215 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:56:01,223 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:56:01,224 DEBUG [RS:0;3456ee6a3164:45735 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:56:01,230 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:56:01,234 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,40477,1731736558105 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:56:01,242 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:56:01,242 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:56:01,243 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:56:01,243 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:56:01,243 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:56:01,243 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,244 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:56:01,244 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,247 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736591247 2024-11-16T05:56:01,249 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:56:01,249 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:56:01,249 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:56:01,250 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:56:01,253 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:56:01,253 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:56:01,253 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:56:01,253 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:56:01,256 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:01,256 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:56:01,255 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35963, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:56:01,258 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:56:01,259 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:56:01,260 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:56:01,262 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:56:01,262 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:56:01,264 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40477 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:01,267 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736561264,5,FailOnTimeoutGroup] 2024-11-16T05:56:01,268 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736561267,5,FailOnTimeoutGroup] 2024-11-16T05:56:01,268 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,268 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:56:01,270 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:56:01,270 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,290 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:56:01,290 WARN [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:56:01,392 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40477,1731736558105 with port=45735, startcode=1731736558702 2024-11-16T05:56:01,396 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40477 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,400 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40477 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,407 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:56:01,407 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45913 2024-11-16T05:56:01,407 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:56:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:56:01,412 DEBUG [RS:0;3456ee6a3164:45735 {}] zookeeper.ZKUtil(111): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,412 WARN [RS:0;3456ee6a3164:45735 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:56:01,412 INFO [RS:0;3456ee6a3164:45735 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:56:01,412 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,414 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,45735,1731736558702] 2024-11-16T05:56:01,438 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:56:01,454 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:56:01,458 INFO [RS:0;3456ee6a3164:45735 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:56:01,458 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,459 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:56:01,464 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:56:01,465 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:56:01,466 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:56:01,467 DEBUG [RS:0;3456ee6a3164:45735 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:56:01,468 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,468 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,468 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,469 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,469 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,469 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,45735,1731736558702-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:56:01,487 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:56:01,489 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,45735,1731736558702-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,489 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,489 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.Replication(171): 3456ee6a3164,45735,1731736558702 started 2024-11-16T05:56:01,506 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:01,507 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,45735,1731736558702, RpcServer on 3456ee6a3164/172.17.0.2:45735, sessionid=0x100471a33620001 2024-11-16T05:56:01,507 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:56:01,508 DEBUG [RS:0;3456ee6a3164:45735 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,508 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,45735,1731736558702' 2024-11-16T05:56:01,508 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:56:01,509 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:56:01,510 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:56:01,510 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:56:01,510 DEBUG [RS:0;3456ee6a3164:45735 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,45735,1731736558702 2024-11-16T05:56:01,510 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,45735,1731736558702' 2024-11-16T05:56:01,510 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:56:01,511 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:56:01,511 DEBUG [RS:0;3456ee6a3164:45735 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:56:01,512 INFO [RS:0;3456ee6a3164:45735 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:56:01,512 INFO [RS:0;3456ee6a3164:45735 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:56:01,619 INFO [RS:0;3456ee6a3164:45735 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:56:01,623 INFO [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C45735%2C1731736558702, suffix=, logDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702, archiveDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/oldWALs, maxLogs=32 2024-11-16T05:56:01,638 DEBUG [RS:0;3456ee6a3164:45735 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702/3456ee6a3164%2C45735%2C1731736558702.1731736561625, exclude list is [], retry=0 2024-11-16T05:56:01,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41419,DS-a1076f56-66f6-48f8-8aac-e785a709d653,DISK] 2024-11-16T05:56:01,646 INFO [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702/3456ee6a3164%2C45735%2C1731736558702.1731736561625 2024-11-16T05:56:01,646 DEBUG [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44519:44519)] 2024-11-16T05:56:01,672 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:56:01,673 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:56:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741833_1009 (size=32) 2024-11-16T05:56:02,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:02,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:56:02,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:56:02,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:56:02,101 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:56:02,101 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:56:02,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:56:02,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:56:02,108 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:56:02,108 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:56:02,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740 2024-11-16T05:56:02,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740 2024-11-16T05:56:02,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:56:02,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:56:02,115 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:56:02,117 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:56:02,122 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:56:02,123 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70380942, jitterRate=0.04875776171684265}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:56:02,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731736562089Initializing all the Stores at 1731736562092 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562092Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562092Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736562092Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562092Cleaning up temporary data from old regions at 1731736562114 (+22 ms)Region opened successfully at 1731736562128 (+14 ms) 2024-11-16T05:56:02,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:56:02,128 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:56:02,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:56:02,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:56:02,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:56:02,130 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:56:02,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736562128Disabling compacts and flushes for region at 1731736562128Disabling writes for close at 1731736562128Writing region close event to WAL at 1731736562129 (+1 ms)Closed at 1731736562130 (+1 ms) 2024-11-16T05:56:02,134 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:56:02,134 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:56:02,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:56:02,146 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:56:02,150 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:56:02,306 DEBUG [3456ee6a3164:40477 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T05:56:02,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:02,325 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,45735,1731736558702, state=OPENING 2024-11-16T05:56:02,330 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:56:02,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:02,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:56:02,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:56:02,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:56:02,334 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:56:02,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:02,512 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:56:02,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43641, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:56:02,528 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:56:02,529 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:56:02,529 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:56:02,532 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C45735%2C1731736558702.meta, suffix=.meta, logDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702, archiveDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/oldWALs, maxLogs=32 2024-11-16T05:56:02,547 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702/3456ee6a3164%2C45735%2C1731736558702.meta.1731736562534.meta, exclude list is [], retry=0 2024-11-16T05:56:02,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41419,DS-a1076f56-66f6-48f8-8aac-e785a709d653,DISK] 2024-11-16T05:56:02,553 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/WALs/3456ee6a3164,45735,1731736558702/3456ee6a3164%2C45735%2C1731736558702.meta.1731736562534.meta 2024-11-16T05:56:02,553 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44519:44519)] 2024-11-16T05:56:02,554 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:56:02,555 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:56:02,558 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:56:02,563 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:56:02,566 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:56:02,567 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:02,567 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:56:02,567 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:56:02,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:56:02,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:56:02,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:56:02,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:56:02,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:56:02,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:56:02,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:56:02,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:56:02,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:56:02,580 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:56:02,582 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740 2024-11-16T05:56:02,585 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740 2024-11-16T05:56:02,588 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:56:02,588 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:56:02,590 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:56:02,593 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:56:02,595 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63882275, jitterRate=-0.048079922795295715}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:56:02,595 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:56:02,596 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736562568Writing region info on filesystem at 1731736562568Initializing all the Stores at 1731736562570 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562570Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562570Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736562570Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736562570Cleaning up temporary data from old regions at 1731736562589 (+19 ms)Running coprocessor post-open hooks at 1731736562595 (+6 ms)Region opened successfully at 1731736562596 (+1 ms) 2024-11-16T05:56:02,602 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731736562505 2024-11-16T05:56:02,614 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:56:02,614 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:56:02,617 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:02,620 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,45735,1731736558702, state=OPEN 2024-11-16T05:56:02,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:56:02,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:56:02,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:56:02,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:56:02,625 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:02,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:56:02,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,45735,1731736558702 in 290 msec 2024-11-16T05:56:02,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:56:02,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 494 msec 2024-11-16T05:56:02,640 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:56:02,640 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:56:02,657 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:02,658 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:02,675 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:02,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:02,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5240 sec 2024-11-16T05:56:02,698 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731736562698, completionTime=-1 2024-11-16T05:56:02,702 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T05:56:02,702 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:56:02,730 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-16T05:56:02,730 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736622730 2024-11-16T05:56:02,730 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736682730 2024-11-16T05:56:02,730 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-16T05:56:02,733 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,733 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,733 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,735 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:40477, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,735 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,736 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,741 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:56:02,762 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.934sec 2024-11-16T05:56:02,763 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:56:02,765 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:56:02,766 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:56:02,766 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:56:02,766 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:56:02,767 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:56:02,768 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:56:02,776 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:56:02,777 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:56:02,777 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40477,1731736558105-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:56:02,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:02,799 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T05:56:02,799 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T05:56:02,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:02,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:02,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:02,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c7a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:02,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:02,826 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:02,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:02,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:02,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:02,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,40477,1731736558105 2024-11-16T05:56:02,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=3604 2024-11-16T05:56:02,878 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:56:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:56:02,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35874ff9 2024-11-16T05:56:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:56:02,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:56:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:56:02,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:56:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:02,921 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:56:02,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T05:56:02,925 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:02,931 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:56:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741835_1011 (size=1061) 2024-11-16T05:56:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:03,349 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:56:03,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741836_1012 (size=53) 2024-11-16T05:56:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:03,763 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:03,763 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d3e0754fed086c5654deac27f4f83969, disabling compactions & flushes 2024-11-16T05:56:03,763 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:03,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:03,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. after waiting 0 ms 2024-11-16T05:56:03,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:03,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:03,765 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3e0754fed086c5654deac27f4f83969: Waiting for close lock at 1731736563763Disabling compacts and flushes for region at 1731736563763Disabling writes for close at 1731736563764 (+1 ms)Writing region close event to WAL at 1731736563764Closed at 1731736563764 2024-11-16T05:56:03,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:56:03,776 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736563770"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736563770"}]},"ts":"1731736563770"} 2024-11-16T05:56:03,782 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:56:03,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:56:03,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736563783"}]},"ts":"1731736563783"} 2024-11-16T05:56:03,790 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:56:03,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, ASSIGN}] 2024-11-16T05:56:03,794 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, ASSIGN 2024-11-16T05:56:03,796 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:56:03,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d3e0754fed086c5654deac27f4f83969, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:03,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, ASSIGN because future has completed 2024-11-16T05:56:03,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:04,124 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:04,124 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:56:04,125 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,125 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:04,125 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,125 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,128 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,131 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:04,132 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3e0754fed086c5654deac27f4f83969 columnFamilyName A 2024-11-16T05:56:04,132 DEBUG [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:04,133 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(327): Store=d3e0754fed086c5654deac27f4f83969/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:04,133 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,135 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:04,136 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3e0754fed086c5654deac27f4f83969 columnFamilyName B 2024-11-16T05:56:04,136 DEBUG [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:04,137 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(327): Store=d3e0754fed086c5654deac27f4f83969/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:04,137 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,139 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:04,139 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3e0754fed086c5654deac27f4f83969 columnFamilyName C 2024-11-16T05:56:04,139 DEBUG [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:04,140 INFO [StoreOpener-d3e0754fed086c5654deac27f4f83969-1 {}] regionserver.HStore(327): Store=d3e0754fed086c5654deac27f4f83969/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:04,141 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:04,141 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,143 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,143 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,145 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,145 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,146 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:56:04,148 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,152 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:56:04,153 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d3e0754fed086c5654deac27f4f83969; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71051941, jitterRate=0.058756425976753235}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:56:04,153 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:04,154 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d3e0754fed086c5654deac27f4f83969: Running coprocessor pre-open hook at 1731736564125Writing region info on filesystem at 1731736564125Initializing all the Stores at 1731736564127 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736564127Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736564128 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736564128Cleaning up temporary data from old regions at 1731736564145 (+17 ms)Running coprocessor post-open hooks at 1731736564153 (+8 ms)Region opened successfully at 1731736564154 (+1 ms) 2024-11-16T05:56:04,156 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., pid=6, masterSystemTime=1731736564114 2024-11-16T05:56:04,159 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:04,159 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:04,160 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d3e0754fed086c5654deac27f4f83969, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:04,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:56:04,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T05:56:04,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 in 208 msec 2024-11-16T05:56:04,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T05:56:04,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, ASSIGN in 380 msec 2024-11-16T05:56:04,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:56:04,177 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736564177"}]},"ts":"1731736564177"} 2024-11-16T05:56:04,180 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:56:04,182 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:56:04,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-11-16T05:56:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:56:05,086 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:05,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,109 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:56:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:56:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-11-16T05:56:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:56:05,116 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:56:05,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,124 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,128 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,135 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5805ae29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37adaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,168 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,174 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:05,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:05,177 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d424882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:05,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d4ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:05,185 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,185 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,185 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,185 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,186 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,186 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,187 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,188 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,191 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:05,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:05,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:05,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:05,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T05:56:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:56:05,213 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:05,216 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:05,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:05,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:56:05,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:05,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/577d96a08b4a4816b51463d6ec64aece is 50, key is test_row_0/A:col10/1731736565231/Put/seqid=0 2024-11-16T05:56:05,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:56:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:05,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741837_1013 (size=12001) 2024-11-16T05:56:05,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/577d96a08b4a4816b51463d6ec64aece 2024-11-16T05:56:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736565337, max timestamp=1731736565402Num uniques 859; 2024-11-16T05:56:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:56:05,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736565362, max timestamp=1731736565399Num uniques 859; 2024-11-16T05:56:05,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:05,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736565362, max timestamp=1731736565402Num uniques 859; 2024-11-16T05:56:05,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736625510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736625516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736625518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736625526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736625522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/21a1c46948fd430c802b34d0be11213c is 50, key is test_row_0/B:col10/1731736565231/Put/seqid=0 2024-11-16T05:56:05,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741838_1014 (size=12001) 2024-11-16T05:56:05,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/21a1c46948fd430c802b34d0be11213c 2024-11-16T05:56:05,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:56:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:05,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:05,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/8cfa11b10b514cb2a921ed930b4704f5 is 50, key is test_row_0/C:col10/1731736565231/Put/seqid=0 2024-11-16T05:56:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741839_1015 (size=12001) 2024-11-16T05:56:05,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/8cfa11b10b514cb2a921ed930b4704f5 2024-11-16T05:56:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736625654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736625654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736625656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/577d96a08b4a4816b51463d6ec64aece as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece 2024-11-16T05:56:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736625655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736625656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T05:56:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/21a1c46948fd430c802b34d0be11213c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c 2024-11-16T05:56:05,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T05:56:05,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/8cfa11b10b514cb2a921ed930b4704f5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5 2024-11-16T05:56:05,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T05:56:05,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T05:56:05,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d3e0754fed086c5654deac27f4f83969 in 460ms, sequenceid=12, compaction requested=false 2024-11-16T05:56:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T05:56:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:05,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T05:56:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:05,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/bf2695bd02164b7b9f68258c599681a6 is 50, key is test_row_0/A:col10/1731736565514/Put/seqid=0 2024-11-16T05:56:05,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741840_1016 (size=12001) 2024-11-16T05:56:05,801 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/bf2695bd02164b7b9f68258c599681a6 2024-11-16T05:56:05,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/b610d546a87e487693687930468ab523 is 50, key is test_row_0/B:col10/1731736565514/Put/seqid=0 2024-11-16T05:56:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:05,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741841_1017 (size=12001) 2024-11-16T05:56:05,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736625903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736625907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736625909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736625909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736625913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736626024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736626024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736626034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736626035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736626038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736626234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736626234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736626243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,253 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/b610d546a87e487693687930468ab523 2024-11-16T05:56:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736626258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736626258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/269c732dae564bbaad5eb011e99bc55a is 50, key is test_row_0/C:col10/1731736565514/Put/seqid=0 2024-11-16T05:56:06,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741842_1018 (size=12001) 2024-11-16T05:56:06,291 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/269c732dae564bbaad5eb011e99bc55a 2024-11-16T05:56:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/bf2695bd02164b7b9f68258c599681a6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6 2024-11-16T05:56:06,326 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T05:56:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/b610d546a87e487693687930468ab523 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523 2024-11-16T05:56:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:06,348 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T05:56:06,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/269c732dae564bbaad5eb011e99bc55a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a 2024-11-16T05:56:06,369 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T05:56:06,372 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d3e0754fed086c5654deac27f4f83969 in 625ms, sequenceid=47, compaction requested=false 2024-11-16T05:56:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T05:56:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T05:56:06,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T05:56:06,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1590 sec 2024-11-16T05:56:06,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.1850 sec 2024-11-16T05:56:06,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:06,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/f4264022399e4081bdb28279089d691d is 50, key is test_row_0/A:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741843_1019 (size=14341) 2024-11-16T05:56:06,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/f4264022399e4081bdb28279089d691d 2024-11-16T05:56:06,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/81294f7a1cd84084a78658f1c1352a09 is 50, key is test_row_0/B:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736626613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736626615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741844_1020 (size=12001) 2024-11-16T05:56:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/81294f7a1cd84084a78658f1c1352a09 2024-11-16T05:56:06,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736566609, max timestamp=1731736566619Num uniques 859; 2024-11-16T05:56:06,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:06,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736566609, max timestamp=1731736566619Num uniques 859; 2024-11-16T05:56:06,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:06,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a4afc4ec0adb46d48decb118b1b2de54 is 50, key is test_row_0/C:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736566609, max timestamp=1731736566638Num uniques 859; 2024-11-16T05:56:06,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741845_1021 (size=12001) 2024-11-16T05:56:06,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a4afc4ec0adb46d48decb118b1b2de54 2024-11-16T05:56:06,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/f4264022399e4081bdb28279089d691d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d 2024-11-16T05:56:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736626713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736626714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736626718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d, entries=200, sequenceid=59, filesize=14.0 K 2024-11-16T05:56:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/81294f7a1cd84084a78658f1c1352a09 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09 2024-11-16T05:56:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736626733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736626733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T05:56:06,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a4afc4ec0adb46d48decb118b1b2de54 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54 2024-11-16T05:56:06,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T05:56:06,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for d3e0754fed086c5654deac27f4f83969 in 222ms, sequenceid=59, compaction requested=true 2024-11-16T05:56:06,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:06,786 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:06,786 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:06,792 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:06,793 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:06,794 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:06,795 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.2 K 2024-11-16T05:56:06,797 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:06,798 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:06,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a1c46948fd430c802b34d0be11213c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736565231 2024-11-16T05:56:06,798 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:06,798 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.4 K 2024-11-16T05:56:06,799 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b610d546a87e487693687930468ab523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731736565510 2024-11-16T05:56:06,800 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 81294f7a1cd84084a78658f1c1352a09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:06,805 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577d96a08b4a4816b51463d6ec64aece, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736565231 2024-11-16T05:56:06,806 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf2695bd02164b7b9f68258c599681a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731736565510 2024-11-16T05:56:06,807 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4264022399e4081bdb28279089d691d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:06,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:06,867 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:06,871 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/83b94b221de64d3bbff5e5e8399aff37 is 50, key is test_row_0/B:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a6451c2765cd4315bf645da661341ec3 is 50, key is test_row_0/A:col10/1731736566837/Put/seqid=0 2024-11-16T05:56:06,875 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#9 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:06,877 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/7b6a54c841074c8a9cbcca10c20823b4 is 50, key is test_row_0/A:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741846_1022 (size=12104) 2024-11-16T05:56:06,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736626879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736626882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736626884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,914 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/83b94b221de64d3bbff5e5e8399aff37 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/83b94b221de64d3bbff5e5e8399aff37 2024-11-16T05:56:06,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741847_1023 (size=14341) 2024-11-16T05:56:06,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a6451c2765cd4315bf645da661341ec3 2024-11-16T05:56:06,939 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 83b94b221de64d3bbff5e5e8399aff37(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:06,939 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:06,942 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736566785; duration=0sec 2024-11-16T05:56:06,943 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:06,943 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:06,943 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:06,946 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:06,946 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:06,946 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:06,947 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.2 K 2024-11-16T05:56:06,947 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cfa11b10b514cb2a921ed930b4704f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736565231 2024-11-16T05:56:06,948 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 269c732dae564bbaad5eb011e99bc55a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731736565510 2024-11-16T05:56:06,951 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a4afc4ec0adb46d48decb118b1b2de54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:06,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741848_1024 (size=12104) 2024-11-16T05:56:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736626953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736626953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:06,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/bfa3e411391546489195ac77e4028a61 is 50, key is test_row_0/B:col10/1731736566837/Put/seqid=0 2024-11-16T05:56:06,976 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/7b6a54c841074c8a9cbcca10c20823b4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/7b6a54c841074c8a9cbcca10c20823b4 2024-11-16T05:56:06,988 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:06,991 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4d682447895645acb557e5ca3a82cdf7 is 50, key is test_row_0/C:col10/1731736565887/Put/seqid=0 2024-11-16T05:56:06,992 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 7b6a54c841074c8a9cbcca10c20823b4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:06,992 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:06,992 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736566776; duration=0sec 2024-11-16T05:56:06,993 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:06,993 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:06,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741849_1025 (size=12001) 2024-11-16T05:56:06,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/bfa3e411391546489195ac77e4028a61 2024-11-16T05:56:07,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741850_1026 (size=12104) 2024-11-16T05:56:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736627003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736627013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/d444d0a48ca84e298526765f353a56ac is 50, key is test_row_0/C:col10/1731736566837/Put/seqid=0 2024-11-16T05:56:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736627013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,036 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4d682447895645acb557e5ca3a82cdf7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4d682447895645acb557e5ca3a82cdf7 2024-11-16T05:56:07,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741851_1027 (size=12001) 2024-11-16T05:56:07,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/d444d0a48ca84e298526765f353a56ac 2024-11-16T05:56:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a6451c2765cd4315bf645da661341ec3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3 2024-11-16T05:56:07,077 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 4d682447895645acb557e5ca3a82cdf7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:07,077 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:07,078 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736566785; duration=0sec 2024-11-16T05:56:07,078 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:07,078 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:07,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3, entries=200, sequenceid=95, filesize=14.0 K 2024-11-16T05:56:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/bfa3e411391546489195ac77e4028a61 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61 2024-11-16T05:56:07,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T05:56:07,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/d444d0a48ca84e298526765f353a56ac as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac 2024-11-16T05:56:07,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T05:56:07,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for d3e0754fed086c5654deac27f4f83969 in 274ms, sequenceid=95, compaction requested=false 2024-11-16T05:56:07,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:07,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:56:07,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:07,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:07,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:07,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:07,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:07,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:07,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/746a223b8c864df4b32150ddb7ed9b10 is 50, key is test_row_0/A:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741852_1028 (size=12001) 2024-11-16T05:56:07,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/746a223b8c864df4b32150ddb7ed9b10 2024-11-16T05:56:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736627284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736627287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736627298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e376ac6860fd4c9180f781ad07113b0f is 50, key is test_row_0/B:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736567252, max timestamp=1731736567297Num uniques 859; 2024-11-16T05:56:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:07,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736567252, max timestamp=1731736567297Num uniques 859; 2024-11-16T05:56:07,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:07,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736567252, max timestamp=1731736567297Num uniques 859; 2024-11-16T05:56:07,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741853_1029 (size=12001) 2024-11-16T05:56:07,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e376ac6860fd4c9180f781ad07113b0f 2024-11-16T05:56:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T05:56:07,354 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:07,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T05:56:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:07,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:07,362 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:07,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:07,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/34cb5935b48b432cba8787fea25089af is 50, key is test_row_0/C:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736627378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736627383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741854_1030 (size=12001) 2024-11-16T05:56:07,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/34cb5935b48b432cba8787fea25089af 2024-11-16T05:56:07,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736627413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736627414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736627414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/746a223b8c864df4b32150ddb7ed9b10 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10 2024-11-16T05:56:07,432 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T05:56:07,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10, entries=150, sequenceid=110, filesize=11.7 K 2024-11-16T05:56:07,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e376ac6860fd4c9180f781ad07113b0f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f 2024-11-16T05:56:07,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f, entries=150, sequenceid=110, filesize=11.7 K 2024-11-16T05:56:07,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/34cb5935b48b432cba8787fea25089af as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af 2024-11-16T05:56:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:07,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af, entries=150, sequenceid=110, filesize=11.7 K 2024-11-16T05:56:07,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for d3e0754fed086c5654deac27f4f83969 in 253ms, sequenceid=110, compaction requested=true 2024-11-16T05:56:07,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:07,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:07,480 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:07,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:07,480 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:07,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:07,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:07,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:07,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:07,484 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:07,484 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:07,484 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,484 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/83b94b221de64d3bbff5e5e8399aff37, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.3 K 2024-11-16T05:56:07,485 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:07,485 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b94b221de64d3bbff5e5e8399aff37, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:07,485 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:07,485 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,485 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/7b6a54c841074c8a9cbcca10c20823b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.5 K 2024-11-16T05:56:07,486 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa3e411391546489195ac77e4028a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731736566699 2024-11-16T05:56:07,486 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6a54c841074c8a9cbcca10c20823b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:07,488 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e376ac6860fd4c9180f781ad07113b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:07,490 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6451c2765cd4315bf645da661341ec3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731736566699 2024-11-16T05:56:07,491 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746a223b8c864df4b32150ddb7ed9b10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:07,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T05:56:07,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:07,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:07,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/598ef46a1dba4651a090a8047c5c2617 is 50, key is test_row_0/A:col10/1731736567357/Put/seqid=0 2024-11-16T05:56:07,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,531 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#19 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:07,532 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/810ae6c50df6406eabb3051ed30badad is 50, key is test_row_0/B:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,535 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:07,536 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/585bd45685f74d698b62e24e2af52652 is 50, key is test_row_0/A:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736627541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736627548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741855_1031 (size=12207) 2024-11-16T05:56:07,568 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/810ae6c50df6406eabb3051ed30badad as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/810ae6c50df6406eabb3051ed30badad 2024-11-16T05:56:07,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741856_1032 (size=12207) 2024-11-16T05:56:07,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741857_1033 (size=14541) 2024-11-16T05:56:07,589 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 810ae6c50df6406eabb3051ed30badad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:07,589 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:07,590 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736567480; duration=0sec 2024-11-16T05:56:07,590 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:07,590 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:07,590 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:07,595 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:07,595 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:07,595 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,595 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4d682447895645acb557e5ca3a82cdf7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.3 K 2024-11-16T05:56:07,596 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d682447895645acb557e5ca3a82cdf7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736565887 2024-11-16T05:56:07,598 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d444d0a48ca84e298526765f353a56ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731736566699 2024-11-16T05:56:07,599 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 34cb5935b48b432cba8787fea25089af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:07,624 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#21 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:07,625 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/87dc03d950884d44ab9096d783c05a2e is 50, key is test_row_0/C:col10/1731736567224/Put/seqid=0 2024-11-16T05:56:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736627633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736627633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736627634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741858_1034 (size=12207) 2024-11-16T05:56:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736627654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736627654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,663 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/87dc03d950884d44ab9096d783c05a2e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/87dc03d950884d44ab9096d783c05a2e 2024-11-16T05:56:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:07,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:07,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:07,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,679 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 87dc03d950884d44ab9096d783c05a2e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:07,680 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:07,680 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736567481; duration=0sec 2024-11-16T05:56:07,680 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:07,680 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:07,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736627863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736627863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:07,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/598ef46a1dba4651a090a8047c5c2617 2024-11-16T05:56:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:07,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:07,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:07,987 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/585bd45685f74d698b62e24e2af52652 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/585bd45685f74d698b62e24e2af52652 2024-11-16T05:56:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,001 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 585bd45685f74d698b62e24e2af52652(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:08,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:08,001 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736567480; duration=0sec 2024-11-16T05:56:08,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:08,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:08,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/6d05a1e2f083477f983b6b7996c6c916 is 50, key is test_row_0/B:col10/1731736567357/Put/seqid=0 2024-11-16T05:56:08,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741859_1035 (size=12151) 2024-11-16T05:56:08,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/6d05a1e2f083477f983b6b7996c6c916 2024-11-16T05:56:08,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0fdf294f0b94dc092004836fc83616b is 50, key is test_row_0/C:col10/1731736567357/Put/seqid=0 2024-11-16T05:56:08,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741860_1036 (size=12151) 2024-11-16T05:56:08,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0fdf294f0b94dc092004836fc83616b 2024-11-16T05:56:08,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/598ef46a1dba4651a090a8047c5c2617 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617 2024-11-16T05:56:08,086 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:56:08,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617, entries=200, sequenceid=146, filesize=14.2 K 2024-11-16T05:56:08,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/6d05a1e2f083477f983b6b7996c6c916 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916 2024-11-16T05:56:08,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T05:56:08,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0fdf294f0b94dc092004836fc83616b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b 2024-11-16T05:56:08,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T05:56:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for d3e0754fed086c5654deac27f4f83969 in 647ms, sequenceid=146, compaction requested=false 2024-11-16T05:56:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:08,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:08,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/9c1721b35a1944d0a276e0d36cc63942 is 50, key is test_row_0/A:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741861_1037 (size=12151) 2024-11-16T05:56:08,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736628311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736628317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736628424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736628424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736628454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736628454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T05:56:08,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T05:56:08,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:56:08,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T05:56:08,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T05:56:08,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T05:56:08,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:56:08,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:56:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736628464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:08,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736628633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736628633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/9c1721b35a1944d0a276e0d36cc63942 2024-11-16T05:56:08,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/8d13083c80b44af599aa2981cae57771 is 50, key is test_row_0/B:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741862_1038 (size=12151) 2024-11-16T05:56:08,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/8d13083c80b44af599aa2981cae57771 2024-11-16T05:56:08,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7b64bbddb72642cb98138a6f73f77f2e is 50, key is test_row_0/C:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741863_1039 (size=12151) 2024-11-16T05:56:08,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7b64bbddb72642cb98138a6f73f77f2e 2024-11-16T05:56:08,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/9c1721b35a1944d0a276e0d36cc63942 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942 2024-11-16T05:56:08,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T05:56:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/8d13083c80b44af599aa2981cae57771 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771 2024-11-16T05:56:08,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:08,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T05:56:08,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7b64bbddb72642cb98138a6f73f77f2e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e 2024-11-16T05:56:08,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T05:56:08,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d3e0754fed086c5654deac27f4f83969 in 614ms, sequenceid=161, compaction requested=true 2024-11-16T05:56:08,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:08,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:08,796 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:08,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:08,796 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:08,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:08,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:08,798 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,798 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/810ae6c50df6406eabb3051ed30badad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.7 K 2024-11-16T05:56:08,799 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:08,799 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:08,799 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,799 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/585bd45685f74d698b62e24e2af52652, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=38.0 K 2024-11-16T05:56:08,799 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 810ae6c50df6406eabb3051ed30badad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:08,800 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585bd45685f74d698b62e24e2af52652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:08,801 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d05a1e2f083477f983b6b7996c6c916, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731736567357 2024-11-16T05:56:08,801 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598ef46a1dba4651a090a8047c5c2617, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731736567357 2024-11-16T05:56:08,803 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d13083c80b44af599aa2981cae57771, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:08,803 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1721b35a1944d0a276e0d36cc63942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:08,826 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:08,827 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/2fae84ea454042fabbb2cc2abcd62db2 is 50, key is test_row_0/B:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,840 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:08,841 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/bed943d00413487492caa61b2ba856a5 is 50, key is test_row_0/A:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741864_1040 (size=12459) 2024-11-16T05:56:08,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741865_1041 (size=12459) 2024-11-16T05:56:08,899 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/bed943d00413487492caa61b2ba856a5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bed943d00413487492caa61b2ba856a5 2024-11-16T05:56:08,916 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into bed943d00413487492caa61b2ba856a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:08,916 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:08,916 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736568796; duration=0sec 2024-11-16T05:56:08,918 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:08,918 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:08,919 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:08,921 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:08,921 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:08,923 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,923 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/87dc03d950884d44ab9096d783c05a2e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.7 K 2024-11-16T05:56:08,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T05:56:08,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:08,924 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87dc03d950884d44ab9096d783c05a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731736566867 2024-11-16T05:56:08,924 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:08,926 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0fdf294f0b94dc092004836fc83616b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731736567357 2024-11-16T05:56:08,929 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b64bbddb72642cb98138a6f73f77f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/42920c5a3b3645ffbc706ca0a20f5144 is 50, key is test_row_0/A:col10/1731736568313/Put/seqid=0 2024-11-16T05:56:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:08,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:08,970 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#30 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:08,971 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/06779a5dca9c49458d382c2f77ab608d is 50, key is test_row_0/C:col10/1731736568178/Put/seqid=0 2024-11-16T05:56:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741866_1042 (size=12151) 2024-11-16T05:56:08,985 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/42920c5a3b3645ffbc706ca0a20f5144 2024-11-16T05:56:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736628989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736628991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741867_1043 (size=12459) 2024-11-16T05:56:09,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/48d096c65e5147c9ae44a51ed23643e6 is 50, key is test_row_0/B:col10/1731736568313/Put/seqid=0 2024-11-16T05:56:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741868_1044 (size=12151) 2024-11-16T05:56:09,045 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/48d096c65e5147c9ae44a51ed23643e6 2024-11-16T05:56:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/79b3177a437c4a61bfeaf506a6f81d5e is 50, key is test_row_0/C:col10/1731736568313/Put/seqid=0 2024-11-16T05:56:09,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741869_1045 (size=12151) 2024-11-16T05:56:09,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/79b3177a437c4a61bfeaf506a6f81d5e 2024-11-16T05:56:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736629104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736629104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/42920c5a3b3645ffbc706ca0a20f5144 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144 2024-11-16T05:56:09,126 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T05:56:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/48d096c65e5147c9ae44a51ed23643e6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6 2024-11-16T05:56:09,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T05:56:09,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/79b3177a437c4a61bfeaf506a6f81d5e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e 2024-11-16T05:56:09,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T05:56:09,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d3e0754fed086c5654deac27f4f83969 in 232ms, sequenceid=182, compaction requested=false 2024-11-16T05:56:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T05:56:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T05:56:09,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T05:56:09,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-11-16T05:56:09,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8100 sec 2024-11-16T05:56:09,262 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/2fae84ea454042fabbb2cc2abcd62db2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2fae84ea454042fabbb2cc2abcd62db2 2024-11-16T05:56:09,281 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 2fae84ea454042fabbb2cc2abcd62db2(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:09,281 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,281 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736568796; duration=0sec 2024-11-16T05:56:09,282 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,282 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:09,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:56:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/09c095980f684fa4ba32cf26875f0dfa is 50, key is test_row_0/A:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741870_1046 (size=14541) 2024-11-16T05:56:09,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/09c095980f684fa4ba32cf26875f0dfa 2024-11-16T05:56:09,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e292e5882da24f75b409117074a74617 is 50, key is test_row_0/B:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741871_1047 (size=12151) 2024-11-16T05:56:09,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e292e5882da24f75b409117074a74617 2024-11-16T05:56:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/03b55f1735ed473c89651d358a49b3f2 is 50, key is test_row_0/C:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736629396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736629395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741872_1048 (size=12151) 2024-11-16T05:56:09,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/03b55f1735ed473c89651d358a49b3f2 2024-11-16T05:56:09,430 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/06779a5dca9c49458d382c2f77ab608d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/06779a5dca9c49458d382c2f77ab608d 2024-11-16T05:56:09,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/09c095980f684fa4ba32cf26875f0dfa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa 2024-11-16T05:56:09,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa, entries=200, sequenceid=197, filesize=14.2 K 2024-11-16T05:56:09,447 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 06779a5dca9c49458d382c2f77ab608d(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:09,447 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,447 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736568797; duration=0sec 2024-11-16T05:56:09,448 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,448 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:09,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e292e5882da24f75b409117074a74617 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617 2024-11-16T05:56:09,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T05:56:09,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/03b55f1735ed473c89651d358a49b3f2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2 2024-11-16T05:56:09,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T05:56:09,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d3e0754fed086c5654deac27f4f83969 in 142ms, sequenceid=197, compaction requested=true 2024-11-16T05:56:09,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,468 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:09,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:09,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,468 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:09,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:09,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:09,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:09,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:09,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:09,475 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:09,475 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:09,475 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,475 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2fae84ea454042fabbb2cc2abcd62db2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.9 K 2024-11-16T05:56:09,476 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:09,476 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:09,476 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,476 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bed943d00413487492caa61b2ba856a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=38.2 K 2024-11-16T05:56:09,476 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fae84ea454042fabbb2cc2abcd62db2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:09,477 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d096c65e5147c9ae44a51ed23643e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731736568292 2024-11-16T05:56:09,477 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting bed943d00413487492caa61b2ba856a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:09,478 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42920c5a3b3645ffbc706ca0a20f5144, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731736568292 2024-11-16T05:56:09,478 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e292e5882da24f75b409117074a74617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568988 2024-11-16T05:56:09,479 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c095980f684fa4ba32cf26875f0dfa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568984 2024-11-16T05:56:09,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/25516a179c2b41dd9c175c9740de2244 is 50, key is test_row_0/A:col10/1731736569351/Put/seqid=0 2024-11-16T05:56:09,504 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:09,504 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:09,504 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/85a2a9b6ca3b4b8d83a20942aac70f4d is 50, key is test_row_0/B:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,504 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/49bb4eab057c4f8f82b5c7da1c41c51f is 50, key is test_row_0/A:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T05:56:09,506 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:09,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T05:56:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:09,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:09,511 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:09,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736629510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736629513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736629513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736629515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736629516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741873_1049 (size=14541) 2024-11-16T05:56:09,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741874_1050 (size=12561) 2024-11-16T05:56:09,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741875_1051 (size=12561) 2024-11-16T05:56:09,583 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/49bb4eab057c4f8f82b5c7da1c41c51f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/49bb4eab057c4f8f82b5c7da1c41c51f 2024-11-16T05:56:09,597 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 49bb4eab057c4f8f82b5c7da1c41c51f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:09,597 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,598 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736569468; duration=0sec 2024-11-16T05:56:09,598 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:09,598 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:09,598 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:09,599 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:09,599 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:09,600 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,600 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/06779a5dca9c49458d382c2f77ab608d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=35.9 K 2024-11-16T05:56:09,600 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06779a5dca9c49458d382c2f77ab608d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731736567513 2024-11-16T05:56:09,602 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b3177a437c4a61bfeaf506a6f81d5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731736568292 2024-11-16T05:56:09,603 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03b55f1735ed473c89651d358a49b3f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568988 2024-11-16T05:56:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:09,616 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:09,617 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/bf084ebf879747efa08cf9c901317888 is 50, key is test_row_0/C:col10/1731736569324/Put/seqid=0 2024-11-16T05:56:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736629623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741876_1052 (size=12561) 2024-11-16T05:56:09,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736629634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,640 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/bf084ebf879747efa08cf9c901317888 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/bf084ebf879747efa08cf9c901317888 2024-11-16T05:56:09,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736629634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,653 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into bf084ebf879747efa08cf9c901317888(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:09,653 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,654 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736569470; duration=0sec 2024-11-16T05:56:09,654 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,655 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:09,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736629735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736629734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:09,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:09,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:09,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736629833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736629843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736629853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:09,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/25516a179c2b41dd9c175c9740de2244 2024-11-16T05:56:09,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e9d19abafb1e4aee96d792df6929c401 is 50, key is test_row_0/B:col10/1731736569351/Put/seqid=0 2024-11-16T05:56:09,967 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/85a2a9b6ca3b4b8d83a20942aac70f4d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/85a2a9b6ca3b4b8d83a20942aac70f4d 2024-11-16T05:56:09,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:09,980 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 85a2a9b6ca3b4b8d83a20942aac70f4d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:09,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,980 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:09,981 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736569468; duration=0sec 2024-11-16T05:56:09,981 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:09,981 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:09,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741877_1053 (size=12151) 2024-11-16T05:56:09,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e9d19abafb1e4aee96d792df6929c401 2024-11-16T05:56:10,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3f01d4df3551416aa44a48f4421883d2 is 50, key is test_row_0/C:col10/1731736569351/Put/seqid=0 2024-11-16T05:56:10,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741878_1054 (size=12151) 2024-11-16T05:56:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736630043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736630043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:10,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736630143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736630153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736630163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3f01d4df3551416aa44a48f4421883d2 2024-11-16T05:56:10,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/25516a179c2b41dd9c175c9740de2244 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244 2024-11-16T05:56:10,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244, entries=200, sequenceid=217, filesize=14.2 K 2024-11-16T05:56:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e9d19abafb1e4aee96d792df6929c401 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401 2024-11-16T05:56:10,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401, entries=150, sequenceid=217, filesize=11.9 K 2024-11-16T05:56:10,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3f01d4df3551416aa44a48f4421883d2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2 2024-11-16T05:56:10,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2, entries=150, sequenceid=217, filesize=11.9 K 2024-11-16T05:56:10,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d3e0754fed086c5654deac27f4f83969 in 1002ms, sequenceid=217, compaction requested=false 2024-11-16T05:56:10,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:10,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:10,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9965829a8b14544a133d0cc5d8ab95a is 50, key is test_row_0/A:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:10,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741879_1055 (size=12151) 2024-11-16T05:56:10,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736630632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736630634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736630653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736630663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736630673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736630743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736630753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:10,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:10,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:10,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736630963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736630964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:10,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9965829a8b14544a133d0cc5d8ab95a 2024-11-16T05:56:10,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/d450c22002b6418da5e371617d2211c0 is 50, key is test_row_0/B:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:11,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741880_1056 (size=12151) 2024-11-16T05:56:11,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/d450c22002b6418da5e371617d2211c0 2024-11-16T05:56:11,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/61553778b06141e6ab89d1511d34f4e8 is 50, key is test_row_0/C:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741881_1057 (size=12151) 2024-11-16T05:56:11,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736631273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736631273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:11,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/61553778b06141e6ab89d1511d34f4e8 2024-11-16T05:56:11,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9965829a8b14544a133d0cc5d8ab95a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a 2024-11-16T05:56:11,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T05:56:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/d450c22002b6418da5e371617d2211c0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0 2024-11-16T05:56:11,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T05:56:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/61553778b06141e6ab89d1511d34f4e8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8 2024-11-16T05:56:11,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T05:56:11,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d3e0754fed086c5654deac27f4f83969 in 932ms, sequenceid=235, compaction requested=true 2024-11-16T05:56:11,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:11,490 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:11,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:11,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:11,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:11,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:11,491 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:11,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:11,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:11,492 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:11,492 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:11,492 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,492 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:11,493 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:11,493 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,493 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/85a2a9b6ca3b4b8d83a20942aac70f4d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.0 K 2024-11-16T05:56:11,493 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/49bb4eab057c4f8f82b5c7da1c41c51f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=38.3 K 2024-11-16T05:56:11,494 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a2a9b6ca3b4b8d83a20942aac70f4d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568988 2024-11-16T05:56:11,494 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 49bb4eab057c4f8f82b5c7da1c41c51f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568988 2024-11-16T05:56:11,495 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9d19abafb1e4aee96d792df6929c401, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731736569351 2024-11-16T05:56:11,495 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 25516a179c2b41dd9c175c9740de2244, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731736569351 2024-11-16T05:56:11,495 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d450c22002b6418da5e371617d2211c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:11,496 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a9965829a8b14544a133d0cc5d8ab95a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:11,513 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:11,514 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/9110a4809af84df0a7e34be0f01d3d9d is 50, key is test_row_0/A:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:11,524 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:11,524 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7df916f76b8a44108f531830db04a126 is 50, key is test_row_0/B:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:11,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T05:56:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,530 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:11,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741882_1058 (size=12663) 2024-11-16T05:56:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a75738f0101d45c0b45e49bf3b162538 is 50, key is test_row_0/A:col10/1731736570631/Put/seqid=0 2024-11-16T05:56:11,558 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/9110a4809af84df0a7e34be0f01d3d9d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9110a4809af84df0a7e34be0f01d3d9d 2024-11-16T05:56:11,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741883_1059 (size=12663) 2024-11-16T05:56:11,572 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 9110a4809af84df0a7e34be0f01d3d9d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:11,572 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:11,572 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736571490; duration=0sec 2024-11-16T05:56:11,572 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:11,572 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:11,573 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:11,575 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7df916f76b8a44108f531830db04a126 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7df916f76b8a44108f531830db04a126 2024-11-16T05:56:11,575 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:11,575 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:11,576 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:11,576 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/bf084ebf879747efa08cf9c901317888, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.0 K 2024-11-16T05:56:11,578 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting bf084ebf879747efa08cf9c901317888, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731736568988 2024-11-16T05:56:11,579 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f01d4df3551416aa44a48f4421883d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731736569351 2024-11-16T05:56:11,581 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 61553778b06141e6ab89d1511d34f4e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:11,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741884_1060 (size=12151) 2024-11-16T05:56:11,584 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 7df916f76b8a44108f531830db04a126(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:11,584 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:11,584 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736571490; duration=0sec 2024-11-16T05:56:11,585 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:11,585 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:11,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a75738f0101d45c0b45e49bf3b162538 2024-11-16T05:56:11,603 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:11,605 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/df4fbae871d7456b8509d7da97576c4a is 50, key is test_row_0/C:col10/1731736570555/Put/seqid=0 2024-11-16T05:56:11,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/9d03dc2eb26340cca2699915ea2d2d04 is 50, key is test_row_0/B:col10/1731736570631/Put/seqid=0 2024-11-16T05:56:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741885_1061 (size=12663) 2024-11-16T05:56:11,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741886_1062 (size=12151) 2024-11-16T05:56:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:11,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736631707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736631715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736631717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736631782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736631782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736631823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736631832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736631833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/9d03dc2eb26340cca2699915ea2d2d04 2024-11-16T05:56:12,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736632033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,043 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/df4fbae871d7456b8509d7da97576c4a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/df4fbae871d7456b8509d7da97576c4a 2024-11-16T05:56:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736632043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736632043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,053 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into df4fbae871d7456b8509d7da97576c4a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:12,053 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,053 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736571491; duration=0sec 2024-11-16T05:56:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2bfe7e8e95d341eb8575363eaff78815 is 50, key is test_row_0/C:col10/1731736570631/Put/seqid=0 2024-11-16T05:56:12,053 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:12,053 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:12,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741887_1063 (size=12151) 2024-11-16T05:56:12,061 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2bfe7e8e95d341eb8575363eaff78815 2024-11-16T05:56:12,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a75738f0101d45c0b45e49bf3b162538 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538 2024-11-16T05:56:12,077 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T05:56:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/9d03dc2eb26340cca2699915ea2d2d04 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04 2024-11-16T05:56:12,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T05:56:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2bfe7e8e95d341eb8575363eaff78815 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815 2024-11-16T05:56:12,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T05:56:12,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for d3e0754fed086c5654deac27f4f83969 in 569ms, sequenceid=252, compaction requested=false 2024-11-16T05:56:12,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:12,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T05:56:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T05:56:12,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T05:56:12,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5910 sec 2024-11-16T05:56:12,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.6000 sec 2024-11-16T05:56:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:12,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/98973945b23841aa97bf802cff242ff8 is 50, key is test_row_0/A:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741888_1064 (size=14741) 2024-11-16T05:56:12,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/98973945b23841aa97bf802cff242ff8 2024-11-16T05:56:12,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/86ec15046205415dbb95aef5db25fb0c is 50, key is test_row_0/B:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736632369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736632370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741889_1065 (size=12301) 2024-11-16T05:56:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736632371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/86ec15046205415dbb95aef5db25fb0c 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3793fd65e68b4e1787f0ed7314768a1a is 50, key is test_row_0/C:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741890_1066 (size=12301) 2024-11-16T05:56:12,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3793fd65e68b4e1787f0ed7314768a1a 2024-11-16T05:56:12,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/98973945b23841aa97bf802cff242ff8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8 2024-11-16T05:56:12,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8, entries=200, sequenceid=272, filesize=14.4 K 2024-11-16T05:56:12,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/86ec15046205415dbb95aef5db25fb0c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c 2024-11-16T05:56:12,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:56:12,425 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-16T05:56:12,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/3793fd65e68b4e1787f0ed7314768a1a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a 2024-11-16T05:56:12,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:56:12,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3e0754fed086c5654deac27f4f83969 in 88ms, sequenceid=272, compaction requested=true 2024-11-16T05:56:12,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,434 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:12,434 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:12,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:12,436 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:12,436 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:12,436 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:12,436 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7df916f76b8a44108f531830db04a126, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.2 K 2024-11-16T05:56:12,437 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:12,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df916f76b8a44108f531830db04a126, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:12,437 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:12,437 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:12,437 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9110a4809af84df0a7e34be0f01d3d9d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=38.6 K 2024-11-16T05:56:12,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d03dc2eb26340cca2699915ea2d2d04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731736570613 2024-11-16T05:56:12,438 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9110a4809af84df0a7e34be0f01d3d9d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:12,438 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ec15046205415dbb95aef5db25fb0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:12,439 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a75738f0101d45c0b45e49bf3b162538, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731736570613 2024-11-16T05:56:12,439 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98973945b23841aa97bf802cff242ff8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:12,450 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:12,450 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/fbf4dcb7387047afa88f0b9c1a6c2526 is 50, key is test_row_0/B:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,453 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:12,454 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/f4fe6f4b3ed947eca415ee7bb638eb56 is 50, key is test_row_0/A:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741892_1068 (size=12915) 2024-11-16T05:56:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:12,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:56:12,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:12,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741891_1067 (size=12915) 2024-11-16T05:56:12,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/92b20f4439d64a3bb9cc6de2221a3ada is 50, key is test_row_0/A:col10/1731736572364/Put/seqid=0 2024-11-16T05:56:12,501 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/fbf4dcb7387047afa88f0b9c1a6c2526 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/fbf4dcb7387047afa88f0b9c1a6c2526 2024-11-16T05:56:12,511 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into fbf4dcb7387047afa88f0b9c1a6c2526(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:12,512 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,512 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736572434; duration=0sec 2024-11-16T05:56:12,512 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:12,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741893_1069 (size=14741) 2024-11-16T05:56:12,512 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:12,512 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:12,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/92b20f4439d64a3bb9cc6de2221a3ada 2024-11-16T05:56:12,514 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:12,514 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:12,514 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:12,514 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/df4fbae871d7456b8509d7da97576c4a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.2 K 2024-11-16T05:56:12,515 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting df4fbae871d7456b8509d7da97576c4a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731736569503 2024-11-16T05:56:12,515 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bfe7e8e95d341eb8575363eaff78815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731736570613 2024-11-16T05:56:12,518 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3793fd65e68b4e1787f0ed7314768a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:12,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736632516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c58100cb1344372bbccb58f7b68a634 is 50, key is test_row_0/B:col10/1731736572364/Put/seqid=0 2024-11-16T05:56:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736632520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736632522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,532 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:12,532 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/ff320cdd783645039405fd1048a87b30 is 50, key is test_row_0/C:col10/1731736571712/Put/seqid=0 2024-11-16T05:56:12,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741894_1070 (size=12301) 2024-11-16T05:56:12,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741895_1071 (size=12915) 2024-11-16T05:56:12,565 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/ff320cdd783645039405fd1048a87b30 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff320cdd783645039405fd1048a87b30 2024-11-16T05:56:12,576 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into ff320cdd783645039405fd1048a87b30(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:12,576 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,576 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736572434; duration=0sec 2024-11-16T05:56:12,576 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:12,576 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:12,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736632633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736632633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736632633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736632793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736632794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,801 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:12,801 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736632857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736632857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736632857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:12,881 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/f4fe6f4b3ed947eca415ee7bb638eb56 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4fe6f4b3ed947eca415ee7bb638eb56 2024-11-16T05:56:12,892 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into f4fe6f4b3ed947eca415ee7bb638eb56(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:12,892 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:12,892 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736572434; duration=0sec 2024-11-16T05:56:12,892 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:12,892 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:12,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c58100cb1344372bbccb58f7b68a634 2024-11-16T05:56:12,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/c47cbcc6f9ef48fb83e1d0b6f879b283 is 50, key is test_row_0/C:col10/1731736572364/Put/seqid=0 2024-11-16T05:56:12,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741896_1072 (size=12301) 2024-11-16T05:56:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736633173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/c47cbcc6f9ef48fb83e1d0b6f879b283 2024-11-16T05:56:13,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/92b20f4439d64a3bb9cc6de2221a3ada as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada 2024-11-16T05:56:13,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada, entries=200, sequenceid=289, filesize=14.4 K 2024-11-16T05:56:13,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c58100cb1344372bbccb58f7b68a634 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634 2024-11-16T05:56:13,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T05:56:13,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/c47cbcc6f9ef48fb83e1d0b6f879b283 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283 2024-11-16T05:56:13,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T05:56:13,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3e0754fed086c5654deac27f4f83969 in 911ms, sequenceid=289, compaction requested=false 2024-11-16T05:56:13,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T05:56:13,663 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:13,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T05:56:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:13,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:13,668 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:13,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:13,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:56:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:13,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:13,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/84b1eb4175b24e5b85e0daea17809d02 is 50, key is test_row_0/A:col10/1731736572519/Put/seqid=0 2024-11-16T05:56:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736633700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736633702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736633708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741897_1073 (size=17181) 2024-11-16T05:56:13,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/84b1eb4175b24e5b85e0daea17809d02 2024-11-16T05:56:13,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c364683eec24227b0fef50b5586ce43 is 50, key is test_row_0/B:col10/1731736572519/Put/seqid=0 2024-11-16T05:56:13,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741898_1074 (size=12301) 2024-11-16T05:56:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:13,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736633823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736633823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:13,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:13,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:13,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:13,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:13,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:13,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736633833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:13,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:13,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:13,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:13,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:13,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:13,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736634033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736634033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736634043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c364683eec24227b0fef50b5586ce43 2024-11-16T05:56:14,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0decb8e7ded434faafe88718abef2b1 is 50, key is test_row_0/C:col10/1731736572519/Put/seqid=0 2024-11-16T05:56:14,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741899_1075 (size=12301) 2024-11-16T05:56:14,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736634343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736634343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736634353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0decb8e7ded434faafe88718abef2b1 2024-11-16T05:56:14,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/84b1eb4175b24e5b85e0daea17809d02 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02 2024-11-16T05:56:14,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02, entries=250, sequenceid=309, filesize=16.8 K 2024-11-16T05:56:14,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/7c364683eec24227b0fef50b5586ce43 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43 2024-11-16T05:56:14,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43, entries=150, sequenceid=309, filesize=12.0 K 2024-11-16T05:56:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/a0decb8e7ded434faafe88718abef2b1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1 2024-11-16T05:56:14,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:14,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1, entries=150, sequenceid=309, filesize=12.0 K 2024-11-16T05:56:14,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3e0754fed086c5654deac27f4f83969 in 925ms, sequenceid=309, compaction requested=true 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:14,606 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:56:14,606 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:14,607 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44837 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:14,607 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:14,607 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:14,607 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:14,607 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,607 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,607 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4fe6f4b3ed947eca415ee7bb638eb56, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=43.8 K 2024-11-16T05:56:14,607 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/fbf4dcb7387047afa88f0b9c1a6c2526, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.6 K 2024-11-16T05:56:14,608 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf4dcb7387047afa88f0b9c1a6c2526, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:14,608 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fe6f4b3ed947eca415ee7bb638eb56, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:14,608 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c58100cb1344372bbccb58f7b68a634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731736572364 2024-11-16T05:56:14,609 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b20f4439d64a3bb9cc6de2221a3ada, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731736572364 2024-11-16T05:56:14,609 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c364683eec24227b0fef50b5586ce43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736572516 2024-11-16T05:56:14,609 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b1eb4175b24e5b85e0daea17809d02, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736572506 2024-11-16T05:56:14,619 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:14,620 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/8502dd0466e0469488fb4e21d949dc91 is 50, key is test_row_0/A:col10/1731736572519/Put/seqid=0 2024-11-16T05:56:14,623 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:14,623 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/f88927d870ca49c4a765272e893ef054 is 50, key is test_row_0/B:col10/1731736572519/Put/seqid=0 2024-11-16T05:56:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741900_1076 (size=13017) 2024-11-16T05:56:14,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741901_1077 (size=13017) 2024-11-16T05:56:14,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T05:56:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,754 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ed05fcad063f476d996afc49a587901f is 50, key is test_row_0/A:col10/1731736573696/Put/seqid=0 2024-11-16T05:56:14,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741902_1078 (size=12301) 2024-11-16T05:56:14,773 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ed05fcad063f476d996afc49a587901f 2024-11-16T05:56:14,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/54ef689b90ba4625b5bf94d069bfdb4e is 50, key is test_row_0/B:col10/1731736573696/Put/seqid=0 2024-11-16T05:56:14,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741903_1079 (size=12301) 2024-11-16T05:56:14,798 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/54ef689b90ba4625b5bf94d069bfdb4e 2024-11-16T05:56:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:14,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2cb25babcd684573849cdbd20e1c5a2b is 50, key is test_row_0/C:col10/1731736573696/Put/seqid=0 2024-11-16T05:56:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:14,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:14,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741904_1080 (size=12301) 2024-11-16T05:56:14,831 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2cb25babcd684573849cdbd20e1c5a2b 2024-11-16T05:56:14,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ed05fcad063f476d996afc49a587901f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f 2024-11-16T05:56:14,852 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T05:56:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/54ef689b90ba4625b5bf94d069bfdb4e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e 2024-11-16T05:56:14,860 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T05:56:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/2cb25babcd684573849cdbd20e1c5a2b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b 2024-11-16T05:56:14,872 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T05:56:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736634865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for d3e0754fed086c5654deac27f4f83969 in 119ms, sequenceid=324, compaction requested=true 2024-11-16T05:56:14,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:14,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:14,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T05:56:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T05:56:14,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T05:56:14,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-16T05:56:14,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2140 sec 2024-11-16T05:56:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:14,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:14,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:14,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5a586622f7ca4fed8d69e135985ca081 is 50, key is test_row_0/A:col10/1731736574881/Put/seqid=0 2024-11-16T05:56:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741905_1081 (size=17181) 2024-11-16T05:56:14,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5a586622f7ca4fed8d69e135985ca081 2024-11-16T05:56:14,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736634903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736634911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736634913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736634913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:14,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11cfddb9fd0d443fa840b21517f1eda4 is 50, key is test_row_0/B:col10/1731736574881/Put/seqid=0 2024-11-16T05:56:14,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741906_1082 (size=12301) 2024-11-16T05:56:14,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11cfddb9fd0d443fa840b21517f1eda4 2024-11-16T05:56:14,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7d8210208b0a4dfc96d71224812bdebf is 50, key is test_row_0/C:col10/1731736574881/Put/seqid=0 2024-11-16T05:56:14,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741907_1083 (size=12301) 2024-11-16T05:56:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736634983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736635023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736635033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736635034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736635034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,058 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/8502dd0466e0469488fb4e21d949dc91 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8502dd0466e0469488fb4e21d949dc91 2024-11-16T05:56:15,064 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/f88927d870ca49c4a765272e893ef054 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f88927d870ca49c4a765272e893ef054 2024-11-16T05:56:15,071 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 8502dd0466e0469488fb4e21d949dc91(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:15,071 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,071 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736574606; duration=0sec 2024-11-16T05:56:15,071 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:15,071 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:15,071 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:15,076 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into f88927d870ca49c4a765272e893ef054(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:15,076 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,076 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736574606; duration=0sec 2024-11-16T05:56:15,076 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:15,076 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:15,076 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:15,076 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:15,076 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:15,077 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff320cdd783645039405fd1048a87b30, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=48.7 K 2024-11-16T05:56:15,078 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff320cdd783645039405fd1048a87b30, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736571704 2024-11-16T05:56:15,080 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c47cbcc6f9ef48fb83e1d0b6f879b283, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731736572364 2024-11-16T05:56:15,081 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0decb8e7ded434faafe88718abef2b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736572516 2024-11-16T05:56:15,082 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb25babcd684573849cdbd20e1c5a2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731736573696 2024-11-16T05:56:15,099 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#71 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:15,100 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/661540bda8a14dc695afbdf09ee832bf is 50, key is test_row_0/C:col10/1731736573696/Put/seqid=0 2024-11-16T05:56:15,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741908_1084 (size=13051) 2024-11-16T05:56:15,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736635193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736635233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736635242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736635243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736635243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7d8210208b0a4dfc96d71224812bdebf 2024-11-16T05:56:15,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5a586622f7ca4fed8d69e135985ca081 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081 2024-11-16T05:56:15,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081, entries=250, sequenceid=344, filesize=16.8 K 2024-11-16T05:56:15,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11cfddb9fd0d443fa840b21517f1eda4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4 2024-11-16T05:56:15,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4, entries=150, sequenceid=344, filesize=12.0 K 2024-11-16T05:56:15,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7d8210208b0a4dfc96d71224812bdebf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf 2024-11-16T05:56:15,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf, entries=150, sequenceid=344, filesize=12.0 K 2024-11-16T05:56:15,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for d3e0754fed086c5654deac27f4f83969 in 513ms, sequenceid=344, compaction requested=true 2024-11-16T05:56:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:15,394 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:15,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:15,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:56:15,396 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:15,396 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:15,396 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:15,396 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8502dd0466e0469488fb4e21d949dc91, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=41.5 K 2024-11-16T05:56:15,396 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8502dd0466e0469488fb4e21d949dc91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736572516 2024-11-16T05:56:15,397 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ed05fcad063f476d996afc49a587901f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731736573696 2024-11-16T05:56:15,397 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a586622f7ca4fed8d69e135985ca081, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731736574865 2024-11-16T05:56:15,408 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:15,408 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/467e06c871aa470b981eef04433359f4 is 50, key is test_row_0/A:col10/1731736574881/Put/seqid=0 2024-11-16T05:56:15,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741909_1085 (size=13119) 2024-11-16T05:56:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:15,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:15,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/05c432a1c61e425c86d05d8341e37552 is 50, key is test_row_0/A:col10/1731736574898/Put/seqid=0 2024-11-16T05:56:15,532 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/661540bda8a14dc695afbdf09ee832bf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/661540bda8a14dc695afbdf09ee832bf 2024-11-16T05:56:15,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741910_1086 (size=14741) 2024-11-16T05:56:15,548 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 661540bda8a14dc695afbdf09ee832bf(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:15,548 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,548 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=12, startTime=1731736574606; duration=0sec 2024-11-16T05:56:15,548 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:56:15,548 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:15,548 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:15,548 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:15,550 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:15,550 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:15,550 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:15,550 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f88927d870ca49c4a765272e893ef054, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.7 K 2024-11-16T05:56:15,551 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88927d870ca49c4a765272e893ef054, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736572516 2024-11-16T05:56:15,552 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ef689b90ba4625b5bf94d069bfdb4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731736573696 2024-11-16T05:56:15,553 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11cfddb9fd0d443fa840b21517f1eda4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731736574865 2024-11-16T05:56:15,563 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:15,564 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/170ae68268524d16bfdf3c1adc53eb65 is 50, key is test_row_0/B:col10/1731736574881/Put/seqid=0 2024-11-16T05:56:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736635566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736635567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736635571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736635572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736635574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741911_1087 (size=13119) 2024-11-16T05:56:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736635683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736635683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736635683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736635684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736635693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T05:56:15,813 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:15,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T05:56:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:15,818 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:15,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:15,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:15,824 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/467e06c871aa470b981eef04433359f4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/467e06c871aa470b981eef04433359f4 2024-11-16T05:56:15,831 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 467e06c871aa470b981eef04433359f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:15,831 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,831 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736575394; duration=0sec 2024-11-16T05:56:15,831 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:15,831 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:15,831 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T05:56:15,832 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T05:56:15,832 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T05:56:15,832 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. because compaction request was cancelled 2024-11-16T05:56:15,833 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:15,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736635893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736635893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736635893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736635893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736635903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:15,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/05c432a1c61e425c86d05d8341e37552 2024-11-16T05:56:15,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3b5d193e4792466b8e2e2390317f1421 is 50, key is test_row_0/B:col10/1731736574898/Put/seqid=0 2024-11-16T05:56:15,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741912_1088 (size=12301) 2024-11-16T05:56:15,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:15,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:15,990 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/170ae68268524d16bfdf3c1adc53eb65 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/170ae68268524d16bfdf3c1adc53eb65 2024-11-16T05:56:15,997 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 170ae68268524d16bfdf3c1adc53eb65(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:15,997 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:15,997 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736575394; duration=0sec 2024-11-16T05:56:15,998 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:15,998 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:16,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736636203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736636203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736636203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736636204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736636213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:16,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3b5d193e4792466b8e2e2390317f1421 2024-11-16T05:56:16,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/b07bad78aff448ec94c10726c2ac7e1b is 50, key is test_row_0/C:col10/1731736574898/Put/seqid=0 2024-11-16T05:56:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741913_1089 (size=12301) 2024-11-16T05:56:16,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:16,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736636713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736636713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736636713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736636714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736636723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:16,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:16,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/b07bad78aff448ec94c10726c2ac7e1b 2024-11-16T05:56:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/05c432a1c61e425c86d05d8341e37552 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552 2024-11-16T05:56:16,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552, entries=200, sequenceid=360, filesize=14.4 K 2024-11-16T05:56:16,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3b5d193e4792466b8e2e2390317f1421 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421 2024-11-16T05:56:16,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T05:56:16,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/b07bad78aff448ec94c10726c2ac7e1b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b 2024-11-16T05:56:16,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T05:56:16,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d3e0754fed086c5654deac27f4f83969 in 1302ms, sequenceid=360, compaction requested=true 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:16,808 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:16,808 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:16,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:16,809 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T05:56:16,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T05:56:16,809 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T05:56:16,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. because compaction request was cancelled 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. because compaction request was cancelled 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:16,810 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:16,811 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,811 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/661540bda8a14dc695afbdf09ee832bf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.8 K 2024-11-16T05:56:16,811 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661540bda8a14dc695afbdf09ee832bf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731736573696 2024-11-16T05:56:16,812 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8210208b0a4dfc96d71224812bdebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731736574865 2024-11-16T05:56:16,813 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07bad78aff448ec94c10726c2ac7e1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731736574898 2024-11-16T05:56:16,822 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:16,823 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/ff7c384b41ea46ffb02bd27f252e83be is 50, key is test_row_0/C:col10/1731736574898/Put/seqid=0 2024-11-16T05:56:16,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741914_1090 (size=13153) 2024-11-16T05:56:16,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T05:56:16,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:16,892 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:16,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/47ed364b949a4b7187184ac731722b36 is 50, key is test_row_0/A:col10/1731736575572/Put/seqid=0 2024-11-16T05:56:16,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741915_1091 (size=12301) 2024-11-16T05:56:16,903 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/47ed364b949a4b7187184ac731722b36 2024-11-16T05:56:16,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e857e9da557a483fb0a67aa6c76c938d is 50, key is test_row_0/B:col10/1731736575572/Put/seqid=0 2024-11-16T05:56:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741916_1092 (size=12301) 2024-11-16T05:56:16,923 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e857e9da557a483fb0a67aa6c76c938d 2024-11-16T05:56:16,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/60eb923b78874407be3f7864ba7a22a1 is 50, key is test_row_0/C:col10/1731736575572/Put/seqid=0 2024-11-16T05:56:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741917_1093 (size=12301) 2024-11-16T05:56:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:17,235 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/ff7c384b41ea46ffb02bd27f252e83be as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff7c384b41ea46ffb02bd27f252e83be 2024-11-16T05:56:17,242 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into ff7c384b41ea46ffb02bd27f252e83be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:17,242 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:17,242 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736576808; duration=0sec 2024-11-16T05:56:17,242 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:17,242 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:17,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/60eb923b78874407be3f7864ba7a22a1 2024-11-16T05:56:17,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/47ed364b949a4b7187184ac731722b36 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36 2024-11-16T05:56:17,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T05:56:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/e857e9da557a483fb0a67aa6c76c938d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d 2024-11-16T05:56:17,369 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T05:56:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/60eb923b78874407be3f7864ba7a22a1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1 2024-11-16T05:56:17,376 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T05:56:17,378 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for d3e0754fed086c5654deac27f4f83969 in 485ms, sequenceid=381, compaction requested=true 2024-11-16T05:56:17,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:17,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:17,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T05:56:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T05:56:17,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T05:56:17,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-16T05:56:17,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5680 sec 2024-11-16T05:56:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:17,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:17,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:17,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5b74c3b23a5c479e9b276b360e79699a is 50, key is test_row_0/A:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:17,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741918_1094 (size=17181) 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736577762, max timestamp=1731736577767Num uniques 859; 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731736577762, max timestamp=1731736577767Num uniques 859; 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736577762, max timestamp=1731736577767Num uniques 859; 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:17,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:17,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736637802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736637805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736637807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736637811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736637813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736637923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736637924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736637925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736637933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736637933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T05:56:17,963 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:17,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T05:56:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:17,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:17,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:17,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:18,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T05:56:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:18,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736638132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736638133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736638133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736638152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736638153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5b74c3b23a5c479e9b276b360e79699a 2024-11-16T05:56:18,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/2529f7a1da3d4688ab696d95b0c47a72 is 50, key is test_row_0/B:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741919_1095 (size=12301) 2024-11-16T05:56:18,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/2529f7a1da3d4688ab696d95b0c47a72 2024-11-16T05:56:18,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/e314521f2ba0464d84a39dddb2d3f91e is 50, key is test_row_0/C:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:18,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741920_1096 (size=12301) 2024-11-16T05:56:18,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/e314521f2ba0464d84a39dddb2d3f91e 2024-11-16T05:56:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5b74c3b23a5c479e9b276b360e79699a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a 2024-11-16T05:56:18,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a, entries=250, sequenceid=392, filesize=16.8 K 2024-11-16T05:56:18,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/2529f7a1da3d4688ab696d95b0c47a72 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72 2024-11-16T05:56:18,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:56:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/e314521f2ba0464d84a39dddb2d3f91e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e 2024-11-16T05:56:18,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:56:18,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d3e0754fed086c5654deac27f4f83969 in 533ms, sequenceid=392, compaction requested=true 2024-11-16T05:56:18,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:18,266 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:18,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:18,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:18,267 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:18,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:18,268 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:18,268 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:18,268 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:18,268 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/467e06c871aa470b981eef04433359f4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=56.0 K 2024-11-16T05:56:18,268 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:18,268 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:18,268 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,268 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/170ae68268524d16bfdf3c1adc53eb65, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=48.8 K 2024-11-16T05:56:18,269 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 170ae68268524d16bfdf3c1adc53eb65, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731736574865 2024-11-16T05:56:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 467e06c871aa470b981eef04433359f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731736574865 2024-11-16T05:56:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c432a1c61e425c86d05d8341e37552, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731736574898 2024-11-16T05:56:18,269 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5d193e4792466b8e2e2390317f1421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731736574898 2024-11-16T05:56:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:18,270 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e857e9da557a483fb0a67aa6c76c938d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731736575569 2024-11-16T05:56:18,270 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47ed364b949a4b7187184ac731722b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731736575569 2024-11-16T05:56:18,270 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b74c3b23a5c479e9b276b360e79699a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:18,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2529f7a1da3d4688ab696d95b0c47a72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:18,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:18,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T05:56:18,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:18,285 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:18,286 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/598bc16f7b0c442aa0929049f7f7abca is 50, key is test_row_0/A:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:18,290 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:18,290 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/9894b40a2b054b9b94fef308fa65cbf5 is 50, key is test_row_0/B:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:18,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ef97fb1b7cb54aa4b157544171360080 is 50, key is test_row_0/A:col10/1731736577809/Put/seqid=0 2024-11-16T05:56:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741921_1097 (size=13255) 2024-11-16T05:56:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741922_1098 (size=13255) 2024-11-16T05:56:18,322 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/598bc16f7b0c442aa0929049f7f7abca as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598bc16f7b0c442aa0929049f7f7abca 2024-11-16T05:56:18,328 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/9894b40a2b054b9b94fef308fa65cbf5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9894b40a2b054b9b94fef308fa65cbf5 2024-11-16T05:56:18,338 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 598bc16f7b0c442aa0929049f7f7abca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:18,338 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:18,339 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=12, startTime=1731736578266; duration=0sec 2024-11-16T05:56:18,339 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:18,339 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:18,339 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:18,342 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741923_1099 (size=12301) 2024-11-16T05:56:18,343 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:18,343 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:18,344 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff7c384b41ea46ffb02bd27f252e83be, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=36.9 K 2024-11-16T05:56:18,344 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 9894b40a2b054b9b94fef308fa65cbf5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:18,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:18,344 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=12, startTime=1731736578267; duration=0sec 2024-11-16T05:56:18,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:18,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:18,344 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff7c384b41ea46ffb02bd27f252e83be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731736574898 2024-11-16T05:56:18,345 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60eb923b78874407be3f7864ba7a22a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731736575569 2024-11-16T05:56:18,345 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e314521f2ba0464d84a39dddb2d3f91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:18,363 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:18,364 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4191a12f2327462cb026e618552a629e is 50, key is test_row_0/C:col10/1731736577728/Put/seqid=0 2024-11-16T05:56:18,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741924_1100 (size=13255) 2024-11-16T05:56:18,376 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4191a12f2327462cb026e618552a629e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4191a12f2327462cb026e618552a629e 2024-11-16T05:56:18,384 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 4191a12f2327462cb026e618552a629e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:18,384 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:18,384 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736578268; duration=0sec 2024-11-16T05:56:18,384 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:18,384 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:18,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736638463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736638463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736638468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736638472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736638474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736638582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736638583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736638583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:18,743 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ef97fb1b7cb54aa4b157544171360080 2024-11-16T05:56:18,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3c68fb429eaf4e0da59fe84c56500114 is 50, key is test_row_0/B:col10/1731736577809/Put/seqid=0 2024-11-16T05:56:18,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741925_1101 (size=12301) 2024-11-16T05:56:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736638793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736638793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736638793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736638982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736638983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:19,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,160 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3c68fb429eaf4e0da59fe84c56500114 2024-11-16T05:56:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7247daa6166641c3850106c28489e991 is 50, key is test_row_0/C:col10/1731736577809/Put/seqid=0 2024-11-16T05:56:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741926_1102 (size=12301) 2024-11-16T05:56:19,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7247daa6166641c3850106c28489e991 2024-11-16T05:56:19,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/ef97fb1b7cb54aa4b157544171360080 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080 2024-11-16T05:56:19,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080, entries=150, sequenceid=428, filesize=12.0 K 2024-11-16T05:56:19,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/3c68fb429eaf4e0da59fe84c56500114 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114 2024-11-16T05:56:19,599 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114, entries=150, sequenceid=428, filesize=12.0 K 2024-11-16T05:56:19,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/7247daa6166641c3850106c28489e991 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991 2024-11-16T05:56:19,606 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991, entries=150, sequenceid=428, filesize=12.0 K 2024-11-16T05:56:19,607 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for d3e0754fed086c5654deac27f4f83969 in 1333ms, sequenceid=428, compaction requested=false 2024-11-16T05:56:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T05:56:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T05:56:19,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T05:56:19,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-16T05:56:19,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6480 sec 2024-11-16T05:56:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:19,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:19,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a1c063e5f0b343e691f79faab5fd2398 is 50, key is test_row_0/A:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:19,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741927_1103 (size=14741) 2024-11-16T05:56:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736639657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736639658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736639664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736639782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736639782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736639782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736639993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736639993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736639993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,997 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736639993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:19,998 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736639993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a1c063e5f0b343e691f79faab5fd2398 2024-11-16T05:56:20,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/f6396109461b4b79aea8ffbb6caf3d12 is 50, key is test_row_0/B:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:20,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741928_1104 (size=12301) 2024-11-16T05:56:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T05:56:20,114 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:20,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T05:56:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:20,116 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:20,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:20,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:20,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:20,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:20,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736640303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736640303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736640303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:20,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/f6396109461b4b79aea8ffbb6caf3d12 2024-11-16T05:56:20,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4b3dda2264b34e0a8053f00e498bbd8c is 50, key is test_row_0/C:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:20,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741929_1105 (size=12301) 2024-11-16T05:56:20,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:20,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736640822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736640823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736640823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:20,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4b3dda2264b34e0a8053f00e498bbd8c 2024-11-16T05:56:20,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a1c063e5f0b343e691f79faab5fd2398 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398 2024-11-16T05:56:20,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398, entries=200, sequenceid=443, filesize=14.4 K 2024-11-16T05:56:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/f6396109461b4b79aea8ffbb6caf3d12 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12 2024-11-16T05:56:20,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:20,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12, entries=150, sequenceid=443, filesize=12.0 K 2024-11-16T05:56:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/4b3dda2264b34e0a8053f00e498bbd8c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c 2024-11-16T05:56:20,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c, entries=150, sequenceid=443, filesize=12.0 K 2024-11-16T05:56:20,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d3e0754fed086c5654deac27f4f83969 in 1277ms, sequenceid=443, compaction requested=true 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:20,896 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:20,896 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:20,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:20,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:20,897 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:20,897 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:20,897 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:20,897 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:20,897 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,897 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:20,897 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598bc16f7b0c442aa0929049f7f7abca, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=39.4 K 2024-11-16T05:56:20,897 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9894b40a2b054b9b94fef308fa65cbf5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.0 K 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9894b40a2b054b9b94fef308fa65cbf5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598bc16f7b0c442aa0929049f7f7abca, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c68fb429eaf4e0da59fe84c56500114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731736577797 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef97fb1b7cb54aa4b157544171360080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731736577797 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f6396109461b4b79aea8ffbb6caf3d12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:20,898 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c063e5f0b343e691f79faab5fd2398, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:20,906 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:20,906 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/12858bef40544deeabc3be7d92cdc94f is 50, key is test_row_0/B:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:20,910 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:20,910 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/170b91ce6e0540688b75b83267f7aee6 is 50, key is test_row_0/A:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:20,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741931_1107 (size=13357) 2024-11-16T05:56:20,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741930_1106 (size=13357) 2024-11-16T05:56:21,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T05:56:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:21,037 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9d6c47a0d9440b696b549205cc1f13f is 50, key is test_row_0/A:col10/1731736579656/Put/seqid=0 2024-11-16T05:56:21,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741932_1108 (size=12301) 2024-11-16T05:56:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:21,325 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/170b91ce6e0540688b75b83267f7aee6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/170b91ce6e0540688b75b83267f7aee6 2024-11-16T05:56:21,332 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/12858bef40544deeabc3be7d92cdc94f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/12858bef40544deeabc3be7d92cdc94f 2024-11-16T05:56:21,333 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 170b91ce6e0540688b75b83267f7aee6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:21,333 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:21,333 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736580896; duration=0sec 2024-11-16T05:56:21,333 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:21,333 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:21,333 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:21,335 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:21,336 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:21,336 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:21,336 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4191a12f2327462cb026e618552a629e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.0 K 2024-11-16T05:56:21,337 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4191a12f2327462cb026e618552a629e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736577724 2024-11-16T05:56:21,337 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7247daa6166641c3850106c28489e991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731736577797 2024-11-16T05:56:21,338 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b3dda2264b34e0a8053f00e498bbd8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:21,341 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 12858bef40544deeabc3be7d92cdc94f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:21,341 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:21,341 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736580896; duration=0sec 2024-11-16T05:56:21,341 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:21,341 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:21,346 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:21,346 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/d52f7adc046648c18d20cf0a3a043c3f is 50, key is test_row_0/C:col10/1731736579615/Put/seqid=0 2024-11-16T05:56:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741933_1109 (size=13357) 2024-11-16T05:56:21,446 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9d6c47a0d9440b696b549205cc1f13f 2024-11-16T05:56:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/c9452ee121a34d6c854896a316d5b6bd is 50, key is test_row_0/B:col10/1731736579656/Put/seqid=0 2024-11-16T05:56:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741934_1110 (size=12301) 2024-11-16T05:56:21,759 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/d52f7adc046648c18d20cf0a3a043c3f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d52f7adc046648c18d20cf0a3a043c3f 2024-11-16T05:56:21,771 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into d52f7adc046648c18d20cf0a3a043c3f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:21,771 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:21,771 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736580896; duration=0sec 2024-11-16T05:56:21,771 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:21,771 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736641855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736641855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736641856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:21,859 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/c9452ee121a34d6c854896a316d5b6bd 2024-11-16T05:56:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/44a36bc1e40d47d2b3c36ec04f0058b4 is 50, key is test_row_0/C:col10/1731736579656/Put/seqid=0 2024-11-16T05:56:21,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741935_1111 (size=12301) 2024-11-16T05:56:21,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/44a36bc1e40d47d2b3c36ec04f0058b4 2024-11-16T05:56:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/a9d6c47a0d9440b696b549205cc1f13f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f 2024-11-16T05:56:21,882 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T05:56:21,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/c9452ee121a34d6c854896a316d5b6bd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd 2024-11-16T05:56:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T05:56:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/44a36bc1e40d47d2b3c36ec04f0058b4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4 2024-11-16T05:56:21,898 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T05:56:21,899 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d3e0754fed086c5654deac27f4f83969 in 862ms, sequenceid=463, compaction requested=false 2024-11-16T05:56:21,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:21,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:21,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T05:56:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T05:56:21,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T05:56:21,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-11-16T05:56:21,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7890 sec 2024-11-16T05:56:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:21,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:21,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:21,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/6add984791c74545b24c63009002150f is 50, key is test_row_0/A:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:21,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741936_1112 (size=14741) 2024-11-16T05:56:21,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736641993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736641993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736641997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736642003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,004 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736642013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,018 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736642103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736642103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736642103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T05:56:22,263 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:22,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T05:56:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:22,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:22,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:22,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736642313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736642313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736642322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:22,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/6add984791c74545b24c63009002150f 2024-11-16T05:56:22,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/410cf71b3d2b4232bbd3ba493728d680 is 50, key is test_row_0/B:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741937_1113 (size=12301) 2024-11-16T05:56:22,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:22,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736642623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736642623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736642632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:22,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/410cf71b3d2b4232bbd3ba493728d680 2024-11-16T05:56:22,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/78a67a0749f1421a8c779ca6bd389450 is 50, key is test_row_0/C:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:22,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741938_1114 (size=12301) 2024-11-16T05:56:22,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:22,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:23,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736643133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736643133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736643143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:23,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:23,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/78a67a0749f1421a8c779ca6bd389450 2024-11-16T05:56:23,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/6add984791c74545b24c63009002150f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f 2024-11-16T05:56:23,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f, entries=200, sequenceid=479, filesize=14.4 K 2024-11-16T05:56:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/410cf71b3d2b4232bbd3ba493728d680 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680 2024-11-16T05:56:23,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680, entries=150, sequenceid=479, filesize=12.0 K 2024-11-16T05:56:23,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/78a67a0749f1421a8c779ca6bd389450 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450 2024-11-16T05:56:23,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450, entries=150, sequenceid=479, filesize=12.0 K 2024-11-16T05:56:23,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d3e0754fed086c5654deac27f4f83969 in 1267ms, sequenceid=479, compaction requested=true 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:23,231 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:23,231 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:23,232 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:23,232 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:23,232 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:23,232 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:23,232 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,232 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/170b91ce6e0540688b75b83267f7aee6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=39.5 K 2024-11-16T05:56:23,233 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 170b91ce6e0540688b75b83267f7aee6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:23,233 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d6c47a0d9440b696b549205cc1f13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731736579642 2024-11-16T05:56:23,234 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,234 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6add984791c74545b24c63009002150f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581845 2024-11-16T05:56:23,234 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/12858bef40544deeabc3be7d92cdc94f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.1 K 2024-11-16T05:56:23,235 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 12858bef40544deeabc3be7d92cdc94f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:23,235 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c9452ee121a34d6c854896a316d5b6bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731736579642 2024-11-16T05:56:23,236 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 410cf71b3d2b4232bbd3ba493728d680, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581851 2024-11-16T05:56:23,242 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:23,242 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:23,243 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/1d64c01a4adc47e78b96ca3516863704 is 50, key is test_row_0/B:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:23,243 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/aae01d822ccd415393a14c2bc15e0e65 is 50, key is test_row_0/A:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:23,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741939_1115 (size=13459) 2024-11-16T05:56:23,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741940_1116 (size=13459) 2024-11-16T05:56:23,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T05:56:23,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,337 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:56:23,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5c35d30ae9ce4c668033c467c387758a is 50, key is test_row_0/A:col10/1731736581991/Put/seqid=0 2024-11-16T05:56:23,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741941_1117 (size=12301) 2024-11-16T05:56:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:23,668 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/1d64c01a4adc47e78b96ca3516863704 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/1d64c01a4adc47e78b96ca3516863704 2024-11-16T05:56:23,672 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/aae01d822ccd415393a14c2bc15e0e65 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/aae01d822ccd415393a14c2bc15e0e65 2024-11-16T05:56:23,673 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 1d64c01a4adc47e78b96ca3516863704(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:23,673 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:23,673 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736583231; duration=0sec 2024-11-16T05:56:23,673 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:23,673 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:23,674 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:23,675 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:23,675 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:23,675 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:23,675 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d52f7adc046648c18d20cf0a3a043c3f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.1 K 2024-11-16T05:56:23,675 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d52f7adc046648c18d20cf0a3a043c3f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731736578454 2024-11-16T05:56:23,676 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 44a36bc1e40d47d2b3c36ec04f0058b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731736579642 2024-11-16T05:56:23,677 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a67a0749f1421a8c779ca6bd389450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581851 2024-11-16T05:56:23,679 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into aae01d822ccd415393a14c2bc15e0e65(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:23,679 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:23,679 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736583231; duration=0sec 2024-11-16T05:56:23,679 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:23,679 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:23,684 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:23,684 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/760e2d61163944899cf99fdcb99d9486 is 50, key is test_row_0/C:col10/1731736581963/Put/seqid=0 2024-11-16T05:56:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741942_1118 (size=13459) 2024-11-16T05:56:23,703 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/760e2d61163944899cf99fdcb99d9486 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/760e2d61163944899cf99fdcb99d9486 2024-11-16T05:56:23,711 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into 760e2d61163944899cf99fdcb99d9486(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:23,711 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:23,711 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736583231; duration=0sec 2024-11-16T05:56:23,711 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:23,711 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:23,747 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5c35d30ae9ce4c668033c467c387758a 2024-11-16T05:56:23,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/25bd9713354946af9f083410bd5d3552 is 50, key is test_row_0/B:col10/1731736581991/Put/seqid=0 2024-11-16T05:56:23,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741943_1119 (size=12301) 2024-11-16T05:56:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. as already flushing 2024-11-16T05:56:24,161 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/25bd9713354946af9f083410bd5d3552 2024-11-16T05:56:24,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/33973aabd57043cd9494afd5f5f0b802 is 50, key is test_row_0/C:col10/1731736581991/Put/seqid=0 2024-11-16T05:56:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741944_1120 (size=12301) 2024-11-16T05:56:24,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736644191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736644199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736644200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736644303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736644313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736644313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736644513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736644523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736644523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/33973aabd57043cd9494afd5f5f0b802 2024-11-16T05:56:24,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/5c35d30ae9ce4c668033c467c387758a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a 2024-11-16T05:56:24,585 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T05:56:24,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/25bd9713354946af9f083410bd5d3552 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552 2024-11-16T05:56:24,590 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T05:56:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/33973aabd57043cd9494afd5f5f0b802 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802 2024-11-16T05:56:24,596 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T05:56:24,597 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d3e0754fed086c5654deac27f4f83969 in 1260ms, sequenceid=498, compaction requested=false 2024-11-16T05:56:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:24,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:24,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T05:56:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T05:56:24,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T05:56:24,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-16T05:56:24,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.3380 sec 2024-11-16T05:56:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:24,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:24,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/da765ca06639456895a7e14ed561a757 is 50, key is test_row_0/A:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741945_1121 (size=14741) 2024-11-16T05:56:24,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736644854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736644858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736644859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736644973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736644973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736644973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736645183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736645183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736645183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,192 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T05:56:25,193 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:25,193 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T05:56:25,193 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:25,195 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T05:56:25,195 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:25,197 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,197 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,197 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,198 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,198 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,198 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:25,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:25,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:25,205 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T05:56:25,205 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:25,205 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,205 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:25,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:25,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/da765ca06639456895a7e14ed561a757 2024-11-16T05:56:25,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/58e12df06b16475499f9b76c628dc0bd is 50, key is test_row_0/B:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:25,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741946_1122 (size=12301) 2024-11-16T05:56:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736645493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736645494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736645493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:25,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/58e12df06b16475499f9b76c628dc0bd 2024-11-16T05:56:25,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/0efa2125735e47158f4dae2062da735e is 50, key is test_row_0/C:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:25,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741947_1123 (size=12301) 2024-11-16T05:56:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39382 deadline: 1731736646004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731736646004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39400 deadline: 1731736646004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:26,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39398 deadline: 1731736646023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:26,025 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39358 deadline: 1731736646033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:26,034 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in d3e0754fed086c5654deac27f4f83969 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:56:26,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/0efa2125735e47158f4dae2062da735e 2024-11-16T05:56:26,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/da765ca06639456895a7e14ed561a757 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757 2024-11-16T05:56:26,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757, entries=200, sequenceid=515, filesize=14.4 K 2024-11-16T05:56:26,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/58e12df06b16475499f9b76c628dc0bd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd 2024-11-16T05:56:26,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd, entries=150, sequenceid=515, filesize=12.0 K 2024-11-16T05:56:26,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/0efa2125735e47158f4dae2062da735e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e 2024-11-16T05:56:26,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e, entries=150, sequenceid=515, filesize=12.0 K 2024-11-16T05:56:26,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d3e0754fed086c5654deac27f4f83969 in 1279ms, sequenceid=515, compaction requested=true 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:26,103 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:26,103 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3e0754fed086c5654deac27f4f83969:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:26,104 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:26,104 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:26,104 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/B is initiating minor compaction (all files) 2024-11-16T05:56:26,104 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/A is initiating minor compaction (all files) 2024-11-16T05:56:26,104 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/B in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:26,104 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/A in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:26,104 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/1d64c01a4adc47e78b96ca3516863704, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.2 K 2024-11-16T05:56:26,104 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/aae01d822ccd415393a14c2bc15e0e65, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=39.6 K 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d64c01a4adc47e78b96ca3516863704, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581851 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae01d822ccd415393a14c2bc15e0e65, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581851 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 25bd9713354946af9f083410bd5d3552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731736581991 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c35d30ae9ce4c668033c467c387758a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731736581991 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e12df06b16475499f9b76c628dc0bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731736584163 2024-11-16T05:56:26,105 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting da765ca06639456895a7e14ed561a757, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731736584163 2024-11-16T05:56:26,112 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:26,113 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/8757d11bdf2d45ac8fd95ba28cd167c0 is 50, key is test_row_0/B:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:26,113 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:26,114 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/8e9ca96e9a314efcb306aa4a7c4eddf7 is 50, key is test_row_0/A:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:26,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741948_1124 (size=13561) 2024-11-16T05:56:26,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741949_1125 (size=13561) 2024-11-16T05:56:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T05:56:26,434 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:26,523 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/8757d11bdf2d45ac8fd95ba28cd167c0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8757d11bdf2d45ac8fd95ba28cd167c0 2024-11-16T05:56:26,523 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/8e9ca96e9a314efcb306aa4a7c4eddf7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8e9ca96e9a314efcb306aa4a7c4eddf7 2024-11-16T05:56:26,530 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/A of d3e0754fed086c5654deac27f4f83969 into 8e9ca96e9a314efcb306aa4a7c4eddf7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:26,530 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/B of d3e0754fed086c5654deac27f4f83969 into 8757d11bdf2d45ac8fd95ba28cd167c0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:26,530 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/A, priority=13, startTime=1731736586103; duration=0sec 2024-11-16T05:56:26,530 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/B, priority=13, startTime=1731736586103; duration=0sec 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:A 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:B 2024-11-16T05:56:26,530 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:26,533 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:26,533 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): d3e0754fed086c5654deac27f4f83969/C is initiating minor compaction (all files) 2024-11-16T05:56:26,533 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3e0754fed086c5654deac27f4f83969/C in TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:26,533 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/760e2d61163944899cf99fdcb99d9486, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp, totalSize=37.2 K 2024-11-16T05:56:26,534 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760e2d61163944899cf99fdcb99d9486, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731736581851 2024-11-16T05:56:26,534 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33973aabd57043cd9494afd5f5f0b802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731736581991 2024-11-16T05:56:26,534 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0efa2125735e47158f4dae2062da735e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731736584163 2024-11-16T05:56:26,544 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3e0754fed086c5654deac27f4f83969#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:26,544 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/f98656f1c55c4f42bf0286c1c485bee6 is 50, key is test_row_0/C:col10/1731736584823/Put/seqid=0 2024-11-16T05:56:26,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741950_1126 (size=13561) 2024-11-16T05:56:26,959 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/f98656f1c55c4f42bf0286c1c485bee6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/f98656f1c55c4f42bf0286c1c485bee6 2024-11-16T05:56:26,965 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3e0754fed086c5654deac27f4f83969/C of d3e0754fed086c5654deac27f4f83969 into f98656f1c55c4f42bf0286c1c485bee6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:26,965 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:26,965 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., storeName=d3e0754fed086c5654deac27f4f83969/C, priority=13, startTime=1731736586103; duration=0sec 2024-11-16T05:56:26,965 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:26,965 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3e0754fed086c5654deac27f4f83969:C 2024-11-16T05:56:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:27,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:56:27,016 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T05:56:27,016 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T05:56:27,017 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:27,017 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:27,017 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,017 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:27,017 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:27,017 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:27,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:27,025 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T05:56:27,025 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:27,025 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,025 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:27,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:27,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/c4595fcec2774e9aa49b33c435fac261 is 50, key is test_row_0/A:col10/1731736587014/Put/seqid=0 2024-11-16T05:56:27,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741951_1127 (size=12301) 2024-11-16T05:56:27,249 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:56:27,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/c4595fcec2774e9aa49b33c435fac261 2024-11-16T05:56:27,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11b1c47a0c984af1bac6313bf3f32cb1 is 50, key is test_row_0/B:col10/1731736587014/Put/seqid=0 2024-11-16T05:56:27,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741952_1128 (size=12301) 2024-11-16T05:56:27,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11b1c47a0c984af1bac6313bf3f32cb1 2024-11-16T05:56:27,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/828cb3542a324d6f8401e99d6afbae87 is 50, key is test_row_0/C:col10/1731736587014/Put/seqid=0 2024-11-16T05:56:27,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741953_1129 (size=12301) 2024-11-16T05:56:28,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/828cb3542a324d6f8401e99d6afbae87 2024-11-16T05:56:28,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/c4595fcec2774e9aa49b33c435fac261 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/c4595fcec2774e9aa49b33c435fac261 2024-11-16T05:56:28,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/c4595fcec2774e9aa49b33c435fac261, entries=150, sequenceid=538, filesize=12.0 K 2024-11-16T05:56:28,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/11b1c47a0c984af1bac6313bf3f32cb1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11b1c47a0c984af1bac6313bf3f32cb1 2024-11-16T05:56:28,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11b1c47a0c984af1bac6313bf3f32cb1, entries=150, sequenceid=538, filesize=12.0 K 2024-11-16T05:56:28,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/828cb3542a324d6f8401e99d6afbae87 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/828cb3542a324d6f8401e99d6afbae87 2024-11-16T05:56:28,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/828cb3542a324d6f8401e99d6afbae87, entries=150, sequenceid=538, filesize=12.0 K 2024-11-16T05:56:28,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for d3e0754fed086c5654deac27f4f83969 in 1293ms, sequenceid=538, compaction requested=false 2024-11-16T05:56:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3e0754fed086c5654deac27f4f83969: 2024-11-16T05:56:36,087 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T05:56:36,087 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:36,087 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,088 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:36,148 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T05:56:36,148 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:56:36,148 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,148 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:36,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:56:36,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-16T05:56:36,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-16T05:56:36,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-16T05:56:36,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-16T05:56:36,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-16T05:56:36,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:56:36,151 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2894 2024-11-16T05:56:36,151 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2884 2024-11-16T05:56:36,151 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:56:36,151 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1245 2024-11-16T05:56:36,151 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3735 rows 2024-11-16T05:56:36,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1256 2024-11-16T05:56:36,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-16T05:56:36,152 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:56:36,153 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:56:36,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:36,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:56:36,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:56:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:36,166 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736596166"}]},"ts":"1731736596166"} 2024-11-16T05:56:36,168 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:56:36,168 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:56:36,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:56:36,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, UNASSIGN}] 2024-11-16T05:56:36,175 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, UNASSIGN 2024-11-16T05:56:36,176 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d3e0754fed086c5654deac27f4f83969, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:36,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, UNASSIGN because future has completed 2024-11-16T05:56:36,179 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:56:36,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:36,338 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:36,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:56:36,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing d3e0754fed086c5654deac27f4f83969, disabling compactions & flushes 2024-11-16T05:56:36,340 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:36,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:36,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. after waiting 0 ms 2024-11-16T05:56:36,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:36,341 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing d3e0754fed086c5654deac27f4f83969 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T05:56:36,341 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=A 2024-11-16T05:56:36,341 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:36,341 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=B 2024-11-16T05:56:36,342 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:36,342 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3e0754fed086c5654deac27f4f83969, store=C 2024-11-16T05:56:36,342 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:36,349 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/dae665b50f3144a2b0b931acd8d8e701 is 50, key is test_row_2/A:col10/1731736596144/Put/seqid=0 2024-11-16T05:56:36,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741954_1130 (size=7415) 2024-11-16T05:56:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:36,755 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/dae665b50f3144a2b0b931acd8d8e701 2024-11-16T05:56:36,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/da6303653ea24173b19feaae3539c5ba is 50, key is test_row_2/B:col10/1731736596144/Put/seqid=0 2024-11-16T05:56:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741955_1131 (size=7415) 2024-11-16T05:56:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:37,175 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/da6303653ea24173b19feaae3539c5ba 2024-11-16T05:56:37,188 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/def8662779ae4d40a3bc3277601f635b is 50, key is test_row_2/C:col10/1731736596144/Put/seqid=0 2024-11-16T05:56:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741956_1132 (size=7415) 2024-11-16T05:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:37,593 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/def8662779ae4d40a3bc3277601f635b 2024-11-16T05:56:37,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/A/dae665b50f3144a2b0b931acd8d8e701 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/dae665b50f3144a2b0b931acd8d8e701 2024-11-16T05:56:37,609 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/dae665b50f3144a2b0b931acd8d8e701, entries=50, sequenceid=544, filesize=7.2 K 2024-11-16T05:56:37,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/B/da6303653ea24173b19feaae3539c5ba as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/da6303653ea24173b19feaae3539c5ba 2024-11-16T05:56:37,615 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/da6303653ea24173b19feaae3539c5ba, entries=50, sequenceid=544, filesize=7.2 K 2024-11-16T05:56:37,616 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/.tmp/C/def8662779ae4d40a3bc3277601f635b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/def8662779ae4d40a3bc3277601f635b 2024-11-16T05:56:37,621 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/def8662779ae4d40a3bc3277601f635b, entries=50, sequenceid=544, filesize=7.2 K 2024-11-16T05:56:37,622 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3e0754fed086c5654deac27f4f83969 in 1282ms, sequenceid=544, compaction requested=true 2024-11-16T05:56:37,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/7b6a54c841074c8a9cbcca10c20823b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/585bd45685f74d698b62e24e2af52652, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bed943d00413487492caa61b2ba856a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/49bb4eab057c4f8f82b5c7da1c41c51f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9110a4809af84df0a7e34be0f01d3d9d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4fe6f4b3ed947eca415ee7bb638eb56, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8502dd0466e0469488fb4e21d949dc91, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/467e06c871aa470b981eef04433359f4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598bc16f7b0c442aa0929049f7f7abca, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/170b91ce6e0540688b75b83267f7aee6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/aae01d822ccd415393a14c2bc15e0e65, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757] to archive 2024-11-16T05:56:37,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:56:37,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/577d96a08b4a4816b51463d6ec64aece 2024-11-16T05:56:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bf2695bd02164b7b9f68258c599681a6 2024-11-16T05:56:37,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4264022399e4081bdb28279089d691d 2024-11-16T05:56:37,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/7b6a54c841074c8a9cbcca10c20823b4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/7b6a54c841074c8a9cbcca10c20823b4 2024-11-16T05:56:37,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a6451c2765cd4315bf645da661341ec3 2024-11-16T05:56:37,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/585bd45685f74d698b62e24e2af52652 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/585bd45685f74d698b62e24e2af52652 2024-11-16T05:56:37,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/746a223b8c864df4b32150ddb7ed9b10 2024-11-16T05:56:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598ef46a1dba4651a090a8047c5c2617 2024-11-16T05:56:37,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bed943d00413487492caa61b2ba856a5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/bed943d00413487492caa61b2ba856a5 2024-11-16T05:56:37,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9c1721b35a1944d0a276e0d36cc63942 2024-11-16T05:56:37,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/42920c5a3b3645ffbc706ca0a20f5144 2024-11-16T05:56:37,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/09c095980f684fa4ba32cf26875f0dfa 2024-11-16T05:56:37,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/49bb4eab057c4f8f82b5c7da1c41c51f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/49bb4eab057c4f8f82b5c7da1c41c51f 2024-11-16T05:56:37,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/25516a179c2b41dd9c175c9740de2244 2024-11-16T05:56:37,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9110a4809af84df0a7e34be0f01d3d9d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/9110a4809af84df0a7e34be0f01d3d9d 2024-11-16T05:56:37,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9965829a8b14544a133d0cc5d8ab95a 2024-11-16T05:56:37,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a75738f0101d45c0b45e49bf3b162538 2024-11-16T05:56:37,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/98973945b23841aa97bf802cff242ff8 2024-11-16T05:56:37,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4fe6f4b3ed947eca415ee7bb638eb56 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/f4fe6f4b3ed947eca415ee7bb638eb56 2024-11-16T05:56:37,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/92b20f4439d64a3bb9cc6de2221a3ada 2024-11-16T05:56:37,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/84b1eb4175b24e5b85e0daea17809d02 2024-11-16T05:56:37,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8502dd0466e0469488fb4e21d949dc91 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8502dd0466e0469488fb4e21d949dc91 2024-11-16T05:56:37,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ed05fcad063f476d996afc49a587901f 2024-11-16T05:56:37,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5a586622f7ca4fed8d69e135985ca081 2024-11-16T05:56:37,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/467e06c871aa470b981eef04433359f4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/467e06c871aa470b981eef04433359f4 2024-11-16T05:56:37,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/05c432a1c61e425c86d05d8341e37552 2024-11-16T05:56:37,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/47ed364b949a4b7187184ac731722b36 2024-11-16T05:56:37,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5b74c3b23a5c479e9b276b360e79699a 2024-11-16T05:56:37,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598bc16f7b0c442aa0929049f7f7abca to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/598bc16f7b0c442aa0929049f7f7abca 2024-11-16T05:56:37,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/ef97fb1b7cb54aa4b157544171360080 2024-11-16T05:56:37,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a1c063e5f0b343e691f79faab5fd2398 2024-11-16T05:56:37,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/170b91ce6e0540688b75b83267f7aee6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/170b91ce6e0540688b75b83267f7aee6 2024-11-16T05:56:37,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/a9d6c47a0d9440b696b549205cc1f13f 2024-11-16T05:56:37,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/6add984791c74545b24c63009002150f 2024-11-16T05:56:37,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/aae01d822ccd415393a14c2bc15e0e65 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/aae01d822ccd415393a14c2bc15e0e65 2024-11-16T05:56:37,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/5c35d30ae9ce4c668033c467c387758a 2024-11-16T05:56:37,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/da765ca06639456895a7e14ed561a757 2024-11-16T05:56:37,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/83b94b221de64d3bbff5e5e8399aff37, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/810ae6c50df6406eabb3051ed30badad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2fae84ea454042fabbb2cc2abcd62db2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/85a2a9b6ca3b4b8d83a20942aac70f4d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7df916f76b8a44108f531830db04a126, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/fbf4dcb7387047afa88f0b9c1a6c2526, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f88927d870ca49c4a765272e893ef054, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/170ae68268524d16bfdf3c1adc53eb65, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9894b40a2b054b9b94fef308fa65cbf5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/12858bef40544deeabc3be7d92cdc94f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/1d64c01a4adc47e78b96ca3516863704, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd] to archive 2024-11-16T05:56:37,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:56:37,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/21a1c46948fd430c802b34d0be11213c 2024-11-16T05:56:37,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/b610d546a87e487693687930468ab523 2024-11-16T05:56:37,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/83b94b221de64d3bbff5e5e8399aff37 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/83b94b221de64d3bbff5e5e8399aff37 2024-11-16T05:56:37,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/81294f7a1cd84084a78658f1c1352a09 2024-11-16T05:56:37,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/bfa3e411391546489195ac77e4028a61 2024-11-16T05:56:37,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/810ae6c50df6406eabb3051ed30badad to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/810ae6c50df6406eabb3051ed30badad 2024-11-16T05:56:37,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e376ac6860fd4c9180f781ad07113b0f 2024-11-16T05:56:37,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/6d05a1e2f083477f983b6b7996c6c916 2024-11-16T05:56:37,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2fae84ea454042fabbb2cc2abcd62db2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2fae84ea454042fabbb2cc2abcd62db2 2024-11-16T05:56:37,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8d13083c80b44af599aa2981cae57771 2024-11-16T05:56:37,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/48d096c65e5147c9ae44a51ed23643e6 2024-11-16T05:56:37,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/85a2a9b6ca3b4b8d83a20942aac70f4d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/85a2a9b6ca3b4b8d83a20942aac70f4d 2024-11-16T05:56:37,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e292e5882da24f75b409117074a74617 2024-11-16T05:56:37,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e9d19abafb1e4aee96d792df6929c401 2024-11-16T05:56:37,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7df916f76b8a44108f531830db04a126 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7df916f76b8a44108f531830db04a126 2024-11-16T05:56:37,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/d450c22002b6418da5e371617d2211c0 2024-11-16T05:56:37,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9d03dc2eb26340cca2699915ea2d2d04 2024-11-16T05:56:37,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/fbf4dcb7387047afa88f0b9c1a6c2526 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/fbf4dcb7387047afa88f0b9c1a6c2526 2024-11-16T05:56:37,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/86ec15046205415dbb95aef5db25fb0c 2024-11-16T05:56:37,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c58100cb1344372bbccb58f7b68a634 2024-11-16T05:56:37,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f88927d870ca49c4a765272e893ef054 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f88927d870ca49c4a765272e893ef054 2024-11-16T05:56:37,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/7c364683eec24227b0fef50b5586ce43 2024-11-16T05:56:37,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/54ef689b90ba4625b5bf94d069bfdb4e 2024-11-16T05:56:37,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/170ae68268524d16bfdf3c1adc53eb65 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/170ae68268524d16bfdf3c1adc53eb65 2024-11-16T05:56:37,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11cfddb9fd0d443fa840b21517f1eda4 2024-11-16T05:56:37,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3b5d193e4792466b8e2e2390317f1421 2024-11-16T05:56:37,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/e857e9da557a483fb0a67aa6c76c938d 2024-11-16T05:56:37,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9894b40a2b054b9b94fef308fa65cbf5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/9894b40a2b054b9b94fef308fa65cbf5 2024-11-16T05:56:37,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/2529f7a1da3d4688ab696d95b0c47a72 2024-11-16T05:56:37,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/3c68fb429eaf4e0da59fe84c56500114 2024-11-16T05:56:37,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/12858bef40544deeabc3be7d92cdc94f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/12858bef40544deeabc3be7d92cdc94f 2024-11-16T05:56:37,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/f6396109461b4b79aea8ffbb6caf3d12 2024-11-16T05:56:37,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/c9452ee121a34d6c854896a316d5b6bd 2024-11-16T05:56:37,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/1d64c01a4adc47e78b96ca3516863704 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/1d64c01a4adc47e78b96ca3516863704 2024-11-16T05:56:37,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/410cf71b3d2b4232bbd3ba493728d680 2024-11-16T05:56:37,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/25bd9713354946af9f083410bd5d3552 2024-11-16T05:56:37,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/58e12df06b16475499f9b76c628dc0bd 2024-11-16T05:56:37,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4d682447895645acb557e5ca3a82cdf7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/87dc03d950884d44ab9096d783c05a2e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/06779a5dca9c49458d382c2f77ab608d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/bf084ebf879747efa08cf9c901317888, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/df4fbae871d7456b8509d7da97576c4a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff320cdd783645039405fd1048a87b30, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/661540bda8a14dc695afbdf09ee832bf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff7c384b41ea46ffb02bd27f252e83be, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4191a12f2327462cb026e618552a629e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d52f7adc046648c18d20cf0a3a043c3f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/760e2d61163944899cf99fdcb99d9486, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e] to archive 2024-11-16T05:56:37,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:56:37,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/8cfa11b10b514cb2a921ed930b4704f5 2024-11-16T05:56:37,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/269c732dae564bbaad5eb011e99bc55a 2024-11-16T05:56:37,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4d682447895645acb557e5ca3a82cdf7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4d682447895645acb557e5ca3a82cdf7 2024-11-16T05:56:37,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a4afc4ec0adb46d48decb118b1b2de54 2024-11-16T05:56:37,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d444d0a48ca84e298526765f353a56ac 2024-11-16T05:56:37,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/87dc03d950884d44ab9096d783c05a2e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/87dc03d950884d44ab9096d783c05a2e 2024-11-16T05:56:37,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/34cb5935b48b432cba8787fea25089af 2024-11-16T05:56:37,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0fdf294f0b94dc092004836fc83616b 2024-11-16T05:56:37,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/06779a5dca9c49458d382c2f77ab608d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/06779a5dca9c49458d382c2f77ab608d 2024-11-16T05:56:37,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7b64bbddb72642cb98138a6f73f77f2e 2024-11-16T05:56:37,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/79b3177a437c4a61bfeaf506a6f81d5e 2024-11-16T05:56:37,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/bf084ebf879747efa08cf9c901317888 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/bf084ebf879747efa08cf9c901317888 2024-11-16T05:56:37,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/03b55f1735ed473c89651d358a49b3f2 2024-11-16T05:56:37,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3f01d4df3551416aa44a48f4421883d2 2024-11-16T05:56:37,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/df4fbae871d7456b8509d7da97576c4a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/df4fbae871d7456b8509d7da97576c4a 2024-11-16T05:56:37,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/61553778b06141e6ab89d1511d34f4e8 2024-11-16T05:56:37,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2bfe7e8e95d341eb8575363eaff78815 2024-11-16T05:56:37,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff320cdd783645039405fd1048a87b30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff320cdd783645039405fd1048a87b30 2024-11-16T05:56:37,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/3793fd65e68b4e1787f0ed7314768a1a 2024-11-16T05:56:37,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/c47cbcc6f9ef48fb83e1d0b6f879b283 2024-11-16T05:56:37,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/a0decb8e7ded434faafe88718abef2b1 2024-11-16T05:56:37,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/661540bda8a14dc695afbdf09ee832bf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/661540bda8a14dc695afbdf09ee832bf 2024-11-16T05:56:37,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/2cb25babcd684573849cdbd20e1c5a2b 2024-11-16T05:56:37,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7d8210208b0a4dfc96d71224812bdebf 2024-11-16T05:56:37,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff7c384b41ea46ffb02bd27f252e83be to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/ff7c384b41ea46ffb02bd27f252e83be 2024-11-16T05:56:37,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/b07bad78aff448ec94c10726c2ac7e1b 2024-11-16T05:56:37,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/60eb923b78874407be3f7864ba7a22a1 2024-11-16T05:56:37,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4191a12f2327462cb026e618552a629e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4191a12f2327462cb026e618552a629e 2024-11-16T05:56:37,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/e314521f2ba0464d84a39dddb2d3f91e 2024-11-16T05:56:37,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/7247daa6166641c3850106c28489e991 2024-11-16T05:56:37,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d52f7adc046648c18d20cf0a3a043c3f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/d52f7adc046648c18d20cf0a3a043c3f 2024-11-16T05:56:37,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/4b3dda2264b34e0a8053f00e498bbd8c 2024-11-16T05:56:37,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/44a36bc1e40d47d2b3c36ec04f0058b4 2024-11-16T05:56:37,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/760e2d61163944899cf99fdcb99d9486 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/760e2d61163944899cf99fdcb99d9486 2024-11-16T05:56:37,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/78a67a0749f1421a8c779ca6bd389450 2024-11-16T05:56:37,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/33973aabd57043cd9494afd5f5f0b802 2024-11-16T05:56:37,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/0efa2125735e47158f4dae2062da735e 2024-11-16T05:56:37,779 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-16T05:56:37,780 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. 2024-11-16T05:56:37,780 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for d3e0754fed086c5654deac27f4f83969: Waiting for close lock at 1731736596340Running coprocessor pre-close hooks at 1731736596340Disabling compacts and flushes for region at 1731736596340Disabling writes for close at 1731736596340Obtaining lock to block concurrent updates at 1731736596341 (+1 ms)Preparing flush snapshotting stores in d3e0754fed086c5654deac27f4f83969 at 1731736596341Finished memstore snapshotting TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731736596342 (+1 ms)Flushing stores of TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969. at 1731736596343 (+1 ms)Flushing d3e0754fed086c5654deac27f4f83969/A: creating writer at 1731736596343Flushing d3e0754fed086c5654deac27f4f83969/A: appending metadata at 1731736596348 (+5 ms)Flushing d3e0754fed086c5654deac27f4f83969/A: closing flushed file at 1731736596348Flushing d3e0754fed086c5654deac27f4f83969/B: creating writer at 1731736596766 (+418 ms)Flushing d3e0754fed086c5654deac27f4f83969/B: appending metadata at 1731736596768 (+2 ms)Flushing d3e0754fed086c5654deac27f4f83969/B: closing flushed file at 1731736596768Flushing d3e0754fed086c5654deac27f4f83969/C: creating writer at 1731736597184 (+416 ms)Flushing d3e0754fed086c5654deac27f4f83969/C: appending metadata at 1731736597187 (+3 ms)Flushing d3e0754fed086c5654deac27f4f83969/C: closing flushed file at 1731736597187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c62e204: reopening flushed file at 1731736597603 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eefcb19: reopening flushed file at 1731736597610 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9d94bc5: reopening flushed file at 1731736597615 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3e0754fed086c5654deac27f4f83969 in 1282ms, sequenceid=544, compaction requested=true at 1731736597622 (+7 ms)Writing region close event to WAL at 1731736597775 (+153 ms)Running coprocessor post-close hooks at 1731736597780 (+5 ms)Closed at 1731736597780 2024-11-16T05:56:37,783 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:37,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d3e0754fed086c5654deac27f4f83969, regionState=CLOSED 2024-11-16T05:56:37,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:56:37,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T05:56:37,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure d3e0754fed086c5654deac27f4f83969, server=3456ee6a3164,45735,1731736558702 in 1.6080 sec 2024-11-16T05:56:37,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-16T05:56:37,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3e0754fed086c5654deac27f4f83969, UNASSIGN in 1.6150 sec 2024-11-16T05:56:37,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T05:56:37,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6220 sec 2024-11-16T05:56:37,795 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736597795"}]},"ts":"1731736597795"} 2024-11-16T05:56:37,797 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:56:37,797 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:56:37,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6370 sec 2024-11-16T05:56:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T05:56:38,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:38,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:56:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,330 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:56:38,332 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,338 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:38,343 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/recovered.edits] 2024-11-16T05:56:38,348 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8e9ca96e9a314efcb306aa4a7c4eddf7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/8e9ca96e9a314efcb306aa4a7c4eddf7 2024-11-16T05:56:38,350 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/c4595fcec2774e9aa49b33c435fac261 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/c4595fcec2774e9aa49b33c435fac261 2024-11-16T05:56:38,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/dae665b50f3144a2b0b931acd8d8e701 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/A/dae665b50f3144a2b0b931acd8d8e701 2024-11-16T05:56:38,356 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11b1c47a0c984af1bac6313bf3f32cb1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/11b1c47a0c984af1bac6313bf3f32cb1 2024-11-16T05:56:38,358 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8757d11bdf2d45ac8fd95ba28cd167c0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/8757d11bdf2d45ac8fd95ba28cd167c0 2024-11-16T05:56:38,360 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/da6303653ea24173b19feaae3539c5ba to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/B/da6303653ea24173b19feaae3539c5ba 2024-11-16T05:56:38,363 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/828cb3542a324d6f8401e99d6afbae87 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/828cb3542a324d6f8401e99d6afbae87 2024-11-16T05:56:38,365 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/def8662779ae4d40a3bc3277601f635b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/def8662779ae4d40a3bc3277601f635b 2024-11-16T05:56:38,367 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/f98656f1c55c4f42bf0286c1c485bee6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/C/f98656f1c55c4f42bf0286c1c485bee6 2024-11-16T05:56:38,372 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/recovered.edits/547.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969/recovered.edits/547.seqid 2024-11-16T05:56:38,372 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/d3e0754fed086c5654deac27f4f83969 2024-11-16T05:56:38,372 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:56:38,379 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T05:56:38,391 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:56:38,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:56:38,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:56:38,422 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736598422"}]},"ts":"9223372036854775807"} 2024-11-16T05:56:38,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:56:38,427 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d3e0754fed086c5654deac27f4f83969, NAME => 'TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:56:38,427 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:56:38,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736598427"}]},"ts":"9223372036854775807"} 2024-11-16T05:56:38,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:56:38,431 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 108 msec 2024-11-16T05:56:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T05:56:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:56:38,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:38,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1574838696_22 at /127.0.0.1:50810 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3456ee6a3164:45735-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 182) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3047 (was 3604) 2024-11-16T05:56:38,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:56:38,465 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=3047 2024-11-16T05:56:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:56:38,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:56:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:38,470 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:56:38,470 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:38,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-16T05:56:38,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:56:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:38,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741957_1133 (size=1061) 2024-11-16T05:56:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:38,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:56:38,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741958_1134 (size=53) 2024-11-16T05:56:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 771e8f3141d9fe1005e8f3db6690cb30, disabling compactions & flushes 2024-11-16T05:56:39,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. after waiting 0 ms 2024-11-16T05:56:39,294 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,295 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 771e8f3141d9fe1005e8f3db6690cb30: Waiting for close lock at 1731736599294Disabling compacts and flushes for region at 1731736599294Disabling writes for close at 1731736599294Writing region close event to WAL at 1731736599294Closed at 1731736599294 2024-11-16T05:56:39,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:56:39,299 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736599298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736599298"}]},"ts":"1731736599298"} 2024-11-16T05:56:39,303 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:56:39,304 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:56:39,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736599304"}]},"ts":"1731736599304"} 2024-11-16T05:56:39,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:56:39,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, ASSIGN}] 2024-11-16T05:56:39,308 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, ASSIGN 2024-11-16T05:56:39,309 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:56:39,461 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:39,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, ASSIGN because future has completed 2024-11-16T05:56:39,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:39,626 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,626 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:56:39,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:39,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,627 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,630 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,632 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:39,633 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName A 2024-11-16T05:56:39,634 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:39,634 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:39,634 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,636 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:39,636 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName B 2024-11-16T05:56:39,636 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:39,637 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:39,637 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,638 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:39,638 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName C 2024-11-16T05:56:39,638 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:39,639 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:39,639 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,639 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,641 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,641 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,642 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:56:39,643 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,645 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:56:39,646 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 771e8f3141d9fe1005e8f3db6690cb30; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73717157, jitterRate=0.09847123920917511}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:56:39,646 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:39,647 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 771e8f3141d9fe1005e8f3db6690cb30: Running coprocessor pre-open hook at 1731736599628Writing region info on filesystem at 1731736599628Initializing all the Stores at 1731736599629 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736599629Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736599630 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736599630Cleaning up temporary data from old regions at 1731736599642 (+12 ms)Running coprocessor post-open hooks at 1731736599646 (+4 ms)Region opened successfully at 1731736599647 (+1 ms) 2024-11-16T05:56:39,647 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., pid=30, masterSystemTime=1731736599620 2024-11-16T05:56:39,649 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,649 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:39,650 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:39,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:56:39,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T05:56:39,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 in 187 msec 2024-11-16T05:56:39,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-16T05:56:39,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, ASSIGN in 349 msec 2024-11-16T05:56:39,659 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:56:39,659 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736599659"}]},"ts":"1731736599659"} 2024-11-16T05:56:39,661 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:56:39,662 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:56:39,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-16T05:56:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-16T05:56:40,625 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:40,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:40,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:40,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:40,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:40,635 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:40,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:40,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:40,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:40,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:56:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:56:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-11-16T05:56:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:56:40,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:56:40,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:56:40,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:56:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:56:40,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741959_1135 (size=1097) 2024-11-16T05:56:41,065 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T05:56:41,065 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T05:56:41,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:56:41,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE}] 2024-11-16T05:56:41,078 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE 2024-11-16T05:56:41,078 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE because future has completed 2024-11-16T05:56:41,081 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:56:41,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:41,235 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:56:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 771e8f3141d9fe1005e8f3db6690cb30, disabling compactions & flushes 2024-11-16T05:56:41,235 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. after waiting 0 ms 2024-11-16T05:56:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,243 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:56:41,244 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,244 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 771e8f3141d9fe1005e8f3db6690cb30: Waiting for close lock at 1731736601235Running coprocessor pre-close hooks at 1731736601235Disabling compacts and flushes for region at 1731736601235Disabling writes for close at 1731736601235Writing region close event to WAL at 1731736601236 (+1 ms)Running coprocessor post-close hooks at 1731736601244 (+8 ms)Closed at 1731736601244 2024-11-16T05:56:41,245 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 771e8f3141d9fe1005e8f3db6690cb30 to self. 2024-11-16T05:56:41,248 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=CLOSED 2024-11-16T05:56:41,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:56:41,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T05:56:41,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 in 173 msec 2024-11-16T05:56:41,257 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=true 2024-11-16T05:56:41,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE because future has completed 2024-11-16T05:56:41,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:56:41,571 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:56:41,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:56:41,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,578 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,580 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:41,588 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName A 2024-11-16T05:56:41,590 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:41,591 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:41,591 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,592 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:41,592 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName B 2024-11-16T05:56:41,592 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:41,592 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:41,592 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,593 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:56:41,593 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 771e8f3141d9fe1005e8f3db6690cb30 columnFamilyName C 2024-11-16T05:56:41,593 DEBUG [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:41,593 INFO [StoreOpener-771e8f3141d9fe1005e8f3db6690cb30-1 {}] regionserver.HStore(327): Store=771e8f3141d9fe1005e8f3db6690cb30/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:56:41,594 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,594 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,594 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,596 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,597 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,597 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,597 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:56:41,598 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,599 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 771e8f3141d9fe1005e8f3db6690cb30; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73925953, jitterRate=0.10158254206180573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:56:41,599 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,602 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 771e8f3141d9fe1005e8f3db6690cb30: Running coprocessor pre-open hook at 1731736601572Writing region info on filesystem at 1731736601572Initializing all the Stores at 1731736601573 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736601574 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736601574Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736601574Cleaning up temporary data from old regions at 1731736601597 (+23 ms)Running coprocessor post-open hooks at 1731736601599 (+2 ms)Region opened successfully at 1731736601602 (+3 ms) 2024-11-16T05:56:41,603 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., pid=35, masterSystemTime=1731736601566 2024-11-16T05:56:41,604 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,605 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,605 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:56:41,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-16T05:56:41,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 in 195 msec 2024-11-16T05:56:41,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-16T05:56:41,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, REOPEN/MOVE in 532 msec 2024-11-16T05:56:41,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T05:56:41,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 545 msec 2024-11-16T05:56:41,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 967 msec 2024-11-16T05:56:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T05:56:41,621 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:41,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce2cd12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3310dfa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be3d499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6122bde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,630 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d2cac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c161e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cab24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,635 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b081a04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,638 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53aa22ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,639 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4276ff37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef94f86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f9eb6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,646 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764a5610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:56:41,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:56:41,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:56:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:56:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:56:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8fefae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:56:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:56:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:56:41,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:56:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:56:41,655 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,655 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,655 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,657 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-16T05:56:41,657 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,657 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:41,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:41,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,659 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,660 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,661 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,666 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:56:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:56:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:56:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:56:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:56:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:41,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dc638ebd7c0c45d583a3de7d6f1f1dff_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736601673/Put/seqid=0 2024-11-16T05:56:41,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736661726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736661728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736601722, max timestamp=1731736601731Num uniques 859; 2024-11-16T05:56:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736601722, max timestamp=1731736601731Num uniques 859; 2024-11-16T05:56:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:41,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736601722, max timestamp=1731736601731Num uniques 859; 2024-11-16T05:56:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:41,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741960_1136 (size=12154) 2024-11-16T05:56:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736661779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736661782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736661783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:41,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:41,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736661844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736661844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736661893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736661894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736661894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:41,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:41,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:41,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:41,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736662053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736662054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736662102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736662103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736662113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:42,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dc638ebd7c0c45d583a3de7d6f1f1dff_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc638ebd7c0c45d583a3de7d6f1f1dff_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:42,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/794041823bd8458294913d9b705abe66, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:42,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/794041823bd8458294913d9b705abe66 is 175, key is test_row_0/A:col10/1731736601673/Put/seqid=0 2024-11-16T05:56:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741961_1137 (size=30955) 2024-11-16T05:56:42,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/794041823bd8458294913d9b705abe66 2024-11-16T05:56:42,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/cf02cb03feae43679b1a45c21f68524d is 50, key is test_row_0/B:col10/1731736601673/Put/seqid=0 2024-11-16T05:56:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741962_1138 (size=12001) 2024-11-16T05:56:42,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/cf02cb03feae43679b1a45c21f68524d 2024-11-16T05:56:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:42,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/a993f528b729462281dd7b7be68f7c09 is 50, key is test_row_0/C:col10/1731736601673/Put/seqid=0 2024-11-16T05:56:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741963_1139 (size=12001) 2024-11-16T05:56:42,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/a993f528b729462281dd7b7be68f7c09 2024-11-16T05:56:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/794041823bd8458294913d9b705abe66 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66 2024-11-16T05:56:42,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66, entries=150, sequenceid=15, filesize=30.2 K 2024-11-16T05:56:42,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/cf02cb03feae43679b1a45c21f68524d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d 2024-11-16T05:56:42,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T05:56:42,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/a993f528b729462281dd7b7be68f7c09 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09 2024-11-16T05:56:42,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T05:56:42,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 771e8f3141d9fe1005e8f3db6690cb30 in 677ms, sequenceid=15, compaction requested=false 2024-11-16T05:56:42,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:42,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:42,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111607120968b4b24163b56aa6f7e49152ff_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736602365/Put/seqid=0 2024-11-16T05:56:42,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741964_1140 (size=17034) 2024-11-16T05:56:42,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:42,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111607120968b4b24163b56aa6f7e49152ff_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111607120968b4b24163b56aa6f7e49152ff_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:42,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0c3847fd2f19435593788573125a2c52, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0c3847fd2f19435593788573125a2c52 is 175, key is test_row_0/A:col10/1731736602365/Put/seqid=0 2024-11-16T05:56:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736662424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736662429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736662435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741965_1141 (size=48139) 2024-11-16T05:56:42,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0c3847fd2f19435593788573125a2c52 2024-11-16T05:56:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736662441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736662443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/36a0eee8ffc14f76a5311c1d7d6d8183 is 50, key is test_row_0/B:col10/1731736602365/Put/seqid=0 2024-11-16T05:56:42,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741966_1142 (size=12001) 2024-11-16T05:56:42,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/36a0eee8ffc14f76a5311c1d7d6d8183 2024-11-16T05:56:42,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b2753840650543479329fa0a20012322 is 50, key is test_row_0/C:col10/1731736602365/Put/seqid=0 2024-11-16T05:56:42,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741967_1143 (size=12001) 2024-11-16T05:56:42,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736662553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736662563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736662564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736662564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736662772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736662783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736662783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736662783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:42,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:42,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:42,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:42,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:42,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b2753840650543479329fa0a20012322 2024-11-16T05:56:42,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0c3847fd2f19435593788573125a2c52 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52 2024-11-16T05:56:42,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52, entries=250, sequenceid=51, filesize=47.0 K 2024-11-16T05:56:42,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/36a0eee8ffc14f76a5311c1d7d6d8183 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183 2024-11-16T05:56:42,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T05:56:42,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b2753840650543479329fa0a20012322 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322 2024-11-16T05:56:42,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T05:56:42,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 771e8f3141d9fe1005e8f3db6690cb30 in 573ms, sequenceid=51, compaction requested=false 2024-11-16T05:56:42,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:42,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:42,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164b72d9cabe234575afcb0f6756b5f35f_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:42,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741968_1144 (size=14594) 2024-11-16T05:56:42,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:43,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164b72d9cabe234575afcb0f6756b5f35f_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b72d9cabe234575afcb0f6756b5f35f_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:43,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/7675cb7fb17b4d68bdbbf6a91ab49803, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:43,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/7675cb7fb17b4d68bdbbf6a91ab49803 is 175, key is test_row_0/A:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:43,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741969_1145 (size=39549) 2024-11-16T05:56:43,020 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/7675cb7fb17b4d68bdbbf6a91ab49803 2024-11-16T05:56:43,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/96b64cc566304624a35ca0c2d3765714 is 50, key is test_row_0/B:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:43,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741970_1146 (size=12001) 2024-11-16T05:56:43,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736603031, max timestamp=1731736603062Num uniques 859; 2024-11-16T05:56:43,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:43,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736603031, max timestamp=1731736603062Num uniques 859; 2024-11-16T05:56:43,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:43,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736603031, max timestamp=1731736603062Num uniques 859; 2024-11-16T05:56:43,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:56:43,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736663100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736663102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736663104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736663105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736663107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736663213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736663223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736663224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736663224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736663224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,319 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:56:43,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736663423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736663433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736663433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736663433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736663433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/96b64cc566304624a35ca0c2d3765714 2024-11-16T05:56:43,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/75dc74d0a03e4fdba6f01edbdc5df52e is 50, key is test_row_0/C:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:43,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741971_1147 (size=12001) 2024-11-16T05:56:43,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,644 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T05:56:43,644 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T05:56:43,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736663733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736663743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736663743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736663743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736663744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:43,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:43,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/75dc74d0a03e4fdba6f01edbdc5df52e 2024-11-16T05:56:43,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/7675cb7fb17b4d68bdbbf6a91ab49803 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803 2024-11-16T05:56:43,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803, entries=200, sequenceid=62, filesize=38.6 K 2024-11-16T05:56:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/96b64cc566304624a35ca0c2d3765714 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714 2024-11-16T05:56:43,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714, entries=150, sequenceid=62, filesize=11.7 K 2024-11-16T05:56:43,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/75dc74d0a03e4fdba6f01edbdc5df52e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e 2024-11-16T05:56:43,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e, entries=150, sequenceid=62, filesize=11.7 K 2024-11-16T05:56:43,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 771e8f3141d9fe1005e8f3db6690cb30 in 936ms, sequenceid=62, compaction requested=true 2024-11-16T05:56:43,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:43,891 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:43,891 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:43,892 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:43,892 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:43,892 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:43,892 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:43,892 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,892 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,892 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=115.9 K 2024-11-16T05:56:43,892 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.2 K 2024-11-16T05:56:43,892 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,893 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803] 2024-11-16T05:56:43,893 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting cf02cb03feae43679b1a45c21f68524d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731736601673 2024-11-16T05:56:43,893 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794041823bd8458294913d9b705abe66, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731736601673 2024-11-16T05:56:43,893 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a0eee8ffc14f76a5311c1d7d6d8183, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731736601761 2024-11-16T05:56:43,894 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3847fd2f19435593788573125a2c52, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731736601759 2024-11-16T05:56:43,894 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b64cc566304624a35ca0c2d3765714, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:43,894 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7675cb7fb17b4d68bdbbf6a91ab49803, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:43,900 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:43,902 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:43,903 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/8ba9a2a573894959b359e7a900e554c6 is 50, key is test_row_0/B:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:43,906 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f138bfeba1ad4c56980cec5d51307f21_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:43,916 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f138bfeba1ad4c56980cec5d51307f21_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:43,917 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f138bfeba1ad4c56980cec5d51307f21_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741973_1149 (size=4469) 2024-11-16T05:56:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741972_1148 (size=12104) 2024-11-16T05:56:43,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-16T05:56:43,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:43,978 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T05:56:43,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:43,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:43,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169b8ad063a120443f9d736fa33acbf951_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736603102/Put/seqid=0 2024-11-16T05:56:44,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741974_1150 (size=12154) 2024-11-16T05:56:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:44,019 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169b8ad063a120443f9d736fa33acbf951_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b8ad063a120443f9d736fa33acbf951_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/dca51fe6d7194497bb2e4de11ca711ea, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/dca51fe6d7194497bb2e4de11ca711ea is 175, key is test_row_0/A:col10/1731736603102/Put/seqid=0 2024-11-16T05:56:44,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741975_1151 (size=30955) 2024-11-16T05:56:44,031 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/dca51fe6d7194497bb2e4de11ca711ea 2024-11-16T05:56:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/022b0247d866415db32311b169925b2f is 50, key is test_row_0/B:col10/1731736603102/Put/seqid=0 2024-11-16T05:56:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741976_1152 (size=12001) 2024-11-16T05:56:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:44,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736664260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736664260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736664260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736664261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736664262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,338 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#130 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:44,339 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/eda201e417c74b27ac6fd14475658eff is 175, key is test_row_0/A:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:44,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741977_1153 (size=31058) 2024-11-16T05:56:44,353 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/8ba9a2a573894959b359e7a900e554c6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/8ba9a2a573894959b359e7a900e554c6 2024-11-16T05:56:44,360 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/eda201e417c74b27ac6fd14475658eff as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff 2024-11-16T05:56:44,361 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into 8ba9a2a573894959b359e7a900e554c6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:44,361 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:44,361 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=13, startTime=1731736603891; duration=0sec 2024-11-16T05:56:44,361 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:44,361 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:56:44,362 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:44,363 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:44,363 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:56:44,363 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:44,363 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.2 K 2024-11-16T05:56:44,364 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a993f528b729462281dd7b7be68f7c09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731736601673 2024-11-16T05:56:44,365 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b2753840650543479329fa0a20012322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731736601761 2024-11-16T05:56:44,367 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 75dc74d0a03e4fdba6f01edbdc5df52e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:44,369 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into eda201e417c74b27ac6fd14475658eff(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:44,369 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:44,369 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=13, startTime=1731736603891; duration=0sec 2024-11-16T05:56:44,369 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:44,369 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:56:44,378 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:44,379 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/7dbd27ed998944958648c8ab61648d7e is 50, key is test_row_0/C:col10/1731736602437/Put/seqid=0 2024-11-16T05:56:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741978_1154 (size=12104) 2024-11-16T05:56:44,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736664383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736664383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736664383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736664383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736664383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,403 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/7dbd27ed998944958648c8ab61648d7e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/7dbd27ed998944958648c8ab61648d7e 2024-11-16T05:56:44,411 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into 7dbd27ed998944958648c8ab61648d7e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:44,412 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:44,412 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=13, startTime=1731736603891; duration=0sec 2024-11-16T05:56:44,413 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:44,413 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:56:44,451 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/022b0247d866415db32311b169925b2f 2024-11-16T05:56:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/71e944360af84da6934ef44050c96da1 is 50, key is test_row_0/C:col10/1731736603102/Put/seqid=0 2024-11-16T05:56:44,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741979_1155 (size=12001) 2024-11-16T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736664607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736664608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736664608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736664608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736664608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,871 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/71e944360af84da6934ef44050c96da1 2024-11-16T05:56:44,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/dca51fe6d7194497bb2e4de11ca711ea as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea 2024-11-16T05:56:44,881 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea, entries=150, sequenceid=98, filesize=30.2 K 2024-11-16T05:56:44,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/022b0247d866415db32311b169925b2f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f 2024-11-16T05:56:44,894 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:56:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/71e944360af84da6934ef44050c96da1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1 2024-11-16T05:56:44,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:56:44,904 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 771e8f3141d9fe1005e8f3db6690cb30 in 925ms, sequenceid=98, compaction requested=false 2024-11-16T05:56:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-16T05:56:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-16T05:56:44,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-16T05:56:44,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2450 sec 2024-11-16T05:56:44,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.2520 sec 2024-11-16T05:56:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:44,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:44,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:44,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163ed48ddfa9144349945d6950681fabfb_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:44,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741980_1156 (size=14594) 2024-11-16T05:56:44,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736664947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736664948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736664948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736664950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:44,964 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163ed48ddfa9144349945d6950681fabfb_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163ed48ddfa9144349945d6950681fabfb_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736664951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:44,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3638e1a08bab460a8fc452269ede4eed, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:44,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3638e1a08bab460a8fc452269ede4eed is 175, key is test_row_0/A:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:44,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741981_1157 (size=39549) 2024-11-16T05:56:44,988 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3638e1a08bab460a8fc452269ede4eed 2024-11-16T05:56:44,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e6d7a458ee9e488980b9c91de8f22f63 is 50, key is test_row_0/B:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:45,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741982_1158 (size=12001) 2024-11-16T05:56:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736665063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736665072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736665073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736665073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736665073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736665273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736665283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736665283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736665283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736665284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e6d7a458ee9e488980b9c91de8f22f63 2024-11-16T05:56:45,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/da748a87e179464da37901ec4cc64195 is 50, key is test_row_0/C:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:45,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741983_1159 (size=12001) 2024-11-16T05:56:45,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/da748a87e179464da37901ec4cc64195 2024-11-16T05:56:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3638e1a08bab460a8fc452269ede4eed as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed 2024-11-16T05:56:45,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed, entries=200, sequenceid=113, filesize=38.6 K 2024-11-16T05:56:45,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e6d7a458ee9e488980b9c91de8f22f63 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63 2024-11-16T05:56:45,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T05:56:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/da748a87e179464da37901ec4cc64195 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195 2024-11-16T05:56:45,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T05:56:45,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 771e8f3141d9fe1005e8f3db6690cb30 in 529ms, sequenceid=113, compaction requested=true 2024-11-16T05:56:45,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:45,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:45,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:45,453 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:45,454 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:45,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:45,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:45,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:45,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:45,456 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:45,456 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:45,456 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,456 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/8ba9a2a573894959b359e7a900e554c6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.3 K 2024-11-16T05:56:45,456 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:45,456 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:45,456 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,456 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba9a2a573894959b359e7a900e554c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:45,457 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=99.2 K 2024-11-16T05:56:45,457 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,457 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed] 2024-11-16T05:56:45,457 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 022b0247d866415db32311b169925b2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736603102 2024-11-16T05:56:45,458 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d7a458ee9e488980b9c91de8f22f63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604259 2024-11-16T05:56:45,458 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda201e417c74b27ac6fd14475658eff, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:45,458 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca51fe6d7194497bb2e4de11ca711ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736603102 2024-11-16T05:56:45,459 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3638e1a08bab460a8fc452269ede4eed, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604258 2024-11-16T05:56:45,470 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:45,470 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:45,470 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/9d04b5039ffa40169f4e5b4ccbdacb0c is 50, key is test_row_0/B:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:45,478 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111607fa047b618d425db3444b738af1cac8_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:45,482 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111607fa047b618d425db3444b738af1cac8_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:45,482 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111607fa047b618d425db3444b738af1cac8_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:45,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741984_1160 (size=12207) 2024-11-16T05:56:45,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741985_1161 (size=4469) 2024-11-16T05:56:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:45,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:45,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165180ccbae6cf42d08214e527f51428b7_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736604949/Put/seqid=0 2024-11-16T05:56:45,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736665598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736665599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736665605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736665607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736665607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741986_1162 (size=14794) 2024-11-16T05:56:45,621 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:45,626 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165180ccbae6cf42d08214e527f51428b7_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165180ccbae6cf42d08214e527f51428b7_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:45,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ba48f2178ee44a97b5505f1c2f6cf158, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:45,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ba48f2178ee44a97b5505f1c2f6cf158 is 175, key is test_row_0/A:col10/1731736604949/Put/seqid=0 2024-11-16T05:56:45,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741987_1163 (size=39749) 2024-11-16T05:56:45,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736665712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736665713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736665713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736665723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736665723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-16T05:56:45,823 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:45,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-16T05:56:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:45,825 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:45,826 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:45,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:45,905 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:45,905 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/59ca5e7efd044614855e93ba9dfcb315 is 175, key is test_row_0/A:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:45,906 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/9d04b5039ffa40169f4e5b4ccbdacb0c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/9d04b5039ffa40169f4e5b4ccbdacb0c 2024-11-16T05:56:45,913 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into 9d04b5039ffa40169f4e5b4ccbdacb0c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:45,913 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:45,913 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=13, startTime=1731736605453; duration=0sec 2024-11-16T05:56:45,913 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:45,913 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:56:45,913 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:45,914 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:45,914 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:56:45,914 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,914 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/7dbd27ed998944958648c8ab61648d7e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.3 K 2024-11-16T05:56:45,915 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dbd27ed998944958648c8ab61648d7e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731736602429 2024-11-16T05:56:45,915 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e944360af84da6934ef44050c96da1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736603102 2024-11-16T05:56:45,915 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting da748a87e179464da37901ec4cc64195, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604259 2024-11-16T05:56:45,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741988_1164 (size=31161) 2024-11-16T05:56:45,927 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:45,927 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/8393fe0665124d399c96852fdfbefcb0 is 50, key is test_row_0/C:col10/1731736604923/Put/seqid=0 2024-11-16T05:56:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736665923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736665923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736665923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:45,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736665933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736665933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:45,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741989_1165 (size=12207) 2024-11-16T05:56:45,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T05:56:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:45,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ba48f2178ee44a97b5505f1c2f6cf158 2024-11-16T05:56:46,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f74ce894d6d343ad9fd3f74b46dc8429 is 50, key is test_row_0/B:col10/1731736604949/Put/seqid=0 2024-11-16T05:56:46,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741990_1166 (size=12151) 2024-11-16T05:56:46,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f74ce894d6d343ad9fd3f74b46dc8429 2024-11-16T05:56:46,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/3acd8cac06eb494db49b095bb057f9cf is 50, key is test_row_0/C:col10/1731736604949/Put/seqid=0 2024-11-16T05:56:46,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741991_1167 (size=12151) 2024-11-16T05:56:46,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T05:56:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:46,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736666232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736666243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736666243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736666243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736666243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T05:56:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/59ca5e7efd044614855e93ba9dfcb315 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315 2024-11-16T05:56:46,332 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into 59ca5e7efd044614855e93ba9dfcb315(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:46,332 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:46,332 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=13, startTime=1731736605453; duration=0sec 2024-11-16T05:56:46,332 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:46,332 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:56:46,351 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/8393fe0665124d399c96852fdfbefcb0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/8393fe0665124d399c96852fdfbefcb0 2024-11-16T05:56:46,357 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into 8393fe0665124d399c96852fdfbefcb0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:46,357 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:46,357 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=13, startTime=1731736605454; duration=0sec 2024-11-16T05:56:46,357 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:46,357 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:56:46,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T05:56:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:46,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/3acd8cac06eb494db49b095bb057f9cf 2024-11-16T05:56:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ba48f2178ee44a97b5505f1c2f6cf158 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158 2024-11-16T05:56:46,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158, entries=200, sequenceid=134, filesize=38.8 K 2024-11-16T05:56:46,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f74ce894d6d343ad9fd3f74b46dc8429 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429 2024-11-16T05:56:46,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429, entries=150, sequenceid=134, filesize=11.9 K 2024-11-16T05:56:46,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/3acd8cac06eb494db49b095bb057f9cf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf 2024-11-16T05:56:46,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf, entries=150, sequenceid=134, filesize=11.9 K 2024-11-16T05:56:46,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 771e8f3141d9fe1005e8f3db6690cb30 in 928ms, sequenceid=134, compaction requested=false 2024-11-16T05:56:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:46,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T05:56:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:46,593 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:56:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c3b876343d74ba79f3dc7b159f99fa9_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736605597/Put/seqid=0 2024-11-16T05:56:46,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741992_1168 (size=12304) 2024-11-16T05:56:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736606769, max timestamp=1731736606783Num uniques 859; 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736606769, max timestamp=1731736606783Num uniques 859; 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736606769, max timestamp=1731736606783Num uniques 859; 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:46,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:56:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736666810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736666812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736666812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736666813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736666821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736666922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736666923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736666933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736666933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736666933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:47,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:47,010 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c3b876343d74ba79f3dc7b159f99fa9_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c3b876343d74ba79f3dc7b159f99fa9_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ec092435fbb34cdf9a68423b59217d5d, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ec092435fbb34cdf9a68423b59217d5d is 175, key is test_row_0/A:col10/1731736605597/Put/seqid=0 2024-11-16T05:56:47,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741993_1169 (size=31105) 2024-11-16T05:56:47,019 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ec092435fbb34cdf9a68423b59217d5d 2024-11-16T05:56:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/47d162bf003a4ee192056007be7c85bd is 50, key is test_row_0/B:col10/1731736605597/Put/seqid=0 2024-11-16T05:56:47,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741994_1170 (size=12151) 2024-11-16T05:56:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736667133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736667133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736667143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736667143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736667143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,440 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/47d162bf003a4ee192056007be7c85bd 2024-11-16T05:56:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736667443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736667443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/0fc68f524c3e40a8b01c233f7828b82a is 50, key is test_row_0/C:col10/1731736605597/Put/seqid=0 2024-11-16T05:56:47,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741995_1171 (size=12151) 2024-11-16T05:56:47,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736667453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736667453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736667453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,852 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/0fc68f524c3e40a8b01c233f7828b82a 2024-11-16T05:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/ec092435fbb34cdf9a68423b59217d5d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d 2024-11-16T05:56:47,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d, entries=150, sequenceid=148, filesize=30.4 K 2024-11-16T05:56:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/47d162bf003a4ee192056007be7c85bd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd 2024-11-16T05:56:47,869 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T05:56:47,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/0fc68f524c3e40a8b01c233f7828b82a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a 2024-11-16T05:56:47,876 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T05:56:47,877 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 771e8f3141d9fe1005e8f3db6690cb30 in 1284ms, sequenceid=148, compaction requested=true 2024-11-16T05:56:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-16T05:56:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-16T05:56:47,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T05:56:47,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0530 sec 2024-11-16T05:56:47,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.0570 sec 2024-11-16T05:56:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:47,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:47,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:47,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164d2513ee0c2b40938c38b5e583fcbf18_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T05:56:47,973 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:47,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:47,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736667963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736667967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T05:56:47,976 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:47,977 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:47,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:47,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736667973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736667974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736667975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:47,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:47,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741996_1172 (size=14794) 2024-11-16T05:56:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736668082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736668082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736668083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736668083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736668083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736668293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736668293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736668293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736668293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736668293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:48,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164d2513ee0c2b40938c38b5e583fcbf18_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d2513ee0c2b40938c38b5e583fcbf18_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:48,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/e9347c47d4b347648a000e276700d39e, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:48,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/e9347c47d4b347648a000e276700d39e is 175, key is test_row_0/A:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:48,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741997_1173 (size=39749) 2024-11-16T05:56:48,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:56:48,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:56:48,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736668603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736668603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736668603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736668603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736668603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:48,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/e9347c47d4b347648a000e276700d39e 2024-11-16T05:56:48,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/1462b56349f64385bfa7f5c1ef98e0a5 is 50, key is test_row_0/B:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:48,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741998_1174 (size=12151) 2024-11-16T05:56:48,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:48,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:48,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736669113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736669114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736669114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736669123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736669123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:49,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/1462b56349f64385bfa7f5c1ef98e0a5 2024-11-16T05:56:49,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/6cb59dd5fc3348bb8c9eafdf73e192cf is 50, key is test_row_0/C:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:49,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741999_1175 (size=12151) 2024-11-16T05:56:49,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:49,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/6cb59dd5fc3348bb8c9eafdf73e192cf 2024-11-16T05:56:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/e9347c47d4b347648a000e276700d39e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e 2024-11-16T05:56:49,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e, entries=200, sequenceid=183, filesize=38.8 K 2024-11-16T05:56:49,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/1462b56349f64385bfa7f5c1ef98e0a5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5 2024-11-16T05:56:49,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T05:56:49,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/6cb59dd5fc3348bb8c9eafdf73e192cf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf 2024-11-16T05:56:49,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T05:56:49,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 771e8f3141d9fe1005e8f3db6690cb30 in 1686ms, sequenceid=183, compaction requested=true 2024-11-16T05:56:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:49,640 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:49,641 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:49,642 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141764 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:49,642 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:49,642 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:49,642 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:49,642 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,642 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,642 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=138.4 K 2024-11-16T05:56:49,642 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/9d04b5039ffa40169f4e5b4ccbdacb0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=47.5 K 2024-11-16T05:56:49,642 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,642 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e] 2024-11-16T05:56:49,643 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d04b5039ffa40169f4e5b4ccbdacb0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604259 2024-11-16T05:56:49,643 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ca5e7efd044614855e93ba9dfcb315, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604259 2024-11-16T05:56:49,643 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f74ce894d6d343ad9fd3f74b46dc8429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731736604947 2024-11-16T05:56:49,643 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba48f2178ee44a97b5505f1c2f6cf158, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731736604946 2024-11-16T05:56:49,644 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d162bf003a4ee192056007be7c85bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736605597 2024-11-16T05:56:49,644 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec092435fbb34cdf9a68423b59217d5d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736605597 2024-11-16T05:56:49,644 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9347c47d4b347648a000e276700d39e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606805 2024-11-16T05:56:49,644 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1462b56349f64385bfa7f5c1ef98e0a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606810 2024-11-16T05:56:49,652 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:49,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T05:56:49,658 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116fde24bc39d1d4852a7c7a81a7f00ed5b_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:49,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:49,660 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:56:49,660 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#151 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:49,660 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/be9ff695e6f84493b900f3e92bdd24b8 is 50, key is test_row_0/B:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:49,663 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116fde24bc39d1d4852a7c7a81a7f00ed5b_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:49,663 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fde24bc39d1d4852a7c7a81a7f00ed5b_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcf8efbb96e5416b887fbb7daabbecf6_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736607966/Put/seqid=0 2024-11-16T05:56:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742000_1176 (size=12493) 2024-11-16T05:56:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742001_1177 (size=4469) 2024-11-16T05:56:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742002_1178 (size=12304) 2024-11-16T05:56:50,095 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:50,096 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0b31865bf2de476abc18392d4b5f1b6e is 175, key is test_row_0/A:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:50,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:50,101 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/be9ff695e6f84493b900f3e92bdd24b8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/be9ff695e6f84493b900f3e92bdd24b8 2024-11-16T05:56:50,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742003_1179 (size=31447) 2024-11-16T05:56:50,106 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcf8efbb96e5416b887fbb7daabbecf6_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcf8efbb96e5416b887fbb7daabbecf6_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:50,107 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into be9ff695e6f84493b900f3e92bdd24b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/24ba7504ba524e809710d29383d22938, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:50,107 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:50,107 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=12, startTime=1731736609641; duration=0sec 2024-11-16T05:56:50,107 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:50,107 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:56:50,107 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/24ba7504ba524e809710d29383d22938 is 175, key is test_row_0/A:col10/1731736607966/Put/seqid=0 2024-11-16T05:56:50,109 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:50,110 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:56:50,110 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:50,110 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/8393fe0665124d399c96852fdfbefcb0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=47.5 K 2024-11-16T05:56:50,110 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8393fe0665124d399c96852fdfbefcb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731736604259 2024-11-16T05:56:50,111 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3acd8cac06eb494db49b095bb057f9cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731736604947 2024-11-16T05:56:50,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742004_1180 (size=31105) 2024-11-16T05:56:50,112 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc68f524c3e40a8b01c233f7828b82a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736605597 2024-11-16T05:56:50,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/24ba7504ba524e809710d29383d22938 2024-11-16T05:56:50,112 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb59dd5fc3348bb8c9eafdf73e192cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606810 2024-11-16T05:56:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:50,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:50,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/0b6d94dcc17b4a1c96f5499416994c13 is 50, key is test_row_0/B:col10/1731736607966/Put/seqid=0 2024-11-16T05:56:50,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742005_1181 (size=12151) 2024-11-16T05:56:50,148 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:50,149 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/fcd695f713af429a8b5dbbaf7d9adb01 is 50, key is test_row_0/C:col10/1731736606810/Put/seqid=0 2024-11-16T05:56:50,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742006_1182 (size=12493) 2024-11-16T05:56:50,160 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/fcd695f713af429a8b5dbbaf7d9adb01 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fcd695f713af429a8b5dbbaf7d9adb01 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731736610151, max timestamp=1731736610160Num uniques 859; 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731736610151, max timestamp=1731736610160Num uniques 859; 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731736610151, max timestamp=1731736610160Num uniques 859; 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:50,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:56:50,172 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into fcd695f713af429a8b5dbbaf7d9adb01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:50,172 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:50,172 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=12, startTime=1731736609641; duration=0sec 2024-11-16T05:56:50,173 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:50,173 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:56:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736670186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736670188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736670188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736670189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736670191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,507 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/0b31865bf2de476abc18392d4b5f1b6e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e 2024-11-16T05:56:50,514 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into 0b31865bf2de476abc18392d4b5f1b6e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:50,514 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:50,514 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=12, startTime=1731736609640; duration=0sec 2024-11-16T05:56:50,514 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:50,514 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:56:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736670513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736670513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736670514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736670514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736670514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,550 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/0b6d94dcc17b4a1c96f5499416994c13 2024-11-16T05:56:50,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/438ad39633c54cf685bc83fc8ee80871 is 50, key is test_row_0/C:col10/1731736607966/Put/seqid=0 2024-11-16T05:56:50,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742007_1183 (size=12151) 2024-11-16T05:56:50,563 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/438ad39633c54cf685bc83fc8ee80871 2024-11-16T05:56:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/24ba7504ba524e809710d29383d22938 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938 2024-11-16T05:56:50,574 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938, entries=150, sequenceid=194, filesize=30.4 K 2024-11-16T05:56:50,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/0b6d94dcc17b4a1c96f5499416994c13 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13 2024-11-16T05:56:50,580 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T05:56:50,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/438ad39633c54cf685bc83fc8ee80871 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871 2024-11-16T05:56:50,586 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T05:56:50,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 771e8f3141d9fe1005e8f3db6690cb30 in 927ms, sequenceid=194, compaction requested=false 2024-11-16T05:56:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T05:56:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T05:56:50,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T05:56:50,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6120 sec 2024-11-16T05:56:50,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.6180 sec 2024-11-16T05:56:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:50,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:50,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636abde86cba24e7ab2097790ce4c8bc4_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736670832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736670830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742008_1184 (size=14794) 2024-11-16T05:56:50,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736670835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736670839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736670839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736670943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736670943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736670953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736670953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736670953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736671153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736671153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736671163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736671163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736671164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:51,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636abde86cba24e7ab2097790ce4c8bc4_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636abde86cba24e7ab2097790ce4c8bc4_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:51,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/a4bae0798d4a47f594437a8ef5870e89, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/a4bae0798d4a47f594437a8ef5870e89 is 175, key is test_row_0/A:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742009_1185 (size=39749) 2024-11-16T05:56:51,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/a4bae0798d4a47f594437a8ef5870e89 2024-11-16T05:56:51,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/018b7acc9c2e4f40a720d4fe766b3627 is 50, key is test_row_0/B:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742010_1186 (size=12151) 2024-11-16T05:56:51,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/018b7acc9c2e4f40a720d4fe766b3627 2024-11-16T05:56:51,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/c00d0725536a4d1099fd4dceaf6a5517 is 50, key is test_row_0/C:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742011_1187 (size=12151) 2024-11-16T05:56:51,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/c00d0725536a4d1099fd4dceaf6a5517 2024-11-16T05:56:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/a4bae0798d4a47f594437a8ef5870e89 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89 2024-11-16T05:56:51,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89, entries=200, sequenceid=233, filesize=38.8 K 2024-11-16T05:56:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/018b7acc9c2e4f40a720d4fe766b3627 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627 2024-11-16T05:56:51,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T05:56:51,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/c00d0725536a4d1099fd4dceaf6a5517 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517 2024-11-16T05:56:51,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T05:56:51,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 771e8f3141d9fe1005e8f3db6690cb30 in 499ms, sequenceid=233, compaction requested=true 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:51,324 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:51,324 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:51,325 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:51,325 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:51,325 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:51,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=99.9 K 2024-11-16T05:56:51,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:51,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89] 2024-11-16T05:56:51,326 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:51,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b31865bf2de476abc18392d4b5f1b6e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606810 2024-11-16T05:56:51,326 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:51,326 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:51,326 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/be9ff695e6f84493b900f3e92bdd24b8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.9 K 2024-11-16T05:56:51,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ba7504ba524e809710d29383d22938, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731736607961 2024-11-16T05:56:51,329 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting be9ff695e6f84493b900f3e92bdd24b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606810 2024-11-16T05:56:51,329 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b6d94dcc17b4a1c96f5499416994c13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731736607961 2024-11-16T05:56:51,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4bae0798d4a47f594437a8ef5870e89, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610182 2024-11-16T05:56:51,332 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 018b7acc9c2e4f40a720d4fe766b3627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610186 2024-11-16T05:56:51,341 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,344 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:51,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c938b634fe5040a09c094b2d9ae5361a is 50, key is test_row_0/B:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,350 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411160d1675033bd944fdb8953a00d12b9e05_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742012_1188 (size=12595) 2024-11-16T05:56:51,352 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411160d1675033bd944fdb8953a00d12b9e05_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,352 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160d1675033bd944fdb8953a00d12b9e05_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742013_1189 (size=4469) 2024-11-16T05:56:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:51,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:51,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f5cd629ab1234eeaab7fa6fecf600b87_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736611464/Put/seqid=0 2024-11-16T05:56:51,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742014_1190 (size=14794) 2024-11-16T05:56:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736671492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736671500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736611481, max timestamp=1731736611502Num uniques 859; 2024-11-16T05:56:51,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:56:51,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736611481, max timestamp=1731736611502Num uniques 859; 2024-11-16T05:56:51,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:56:51,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736611481, max timestamp=1731736611502Num uniques 859; 2024-11-16T05:56:51,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:56:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736671539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736671541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736671541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736671613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736671613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736671653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736671653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736671653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,758 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c938b634fe5040a09c094b2d9ae5361a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c938b634fe5040a09c094b2d9ae5361a 2024-11-16T05:56:51,763 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into c938b634fe5040a09c094b2d9ae5361a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:51,763 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:51,763 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=13, startTime=1731736611324; duration=0sec 2024-11-16T05:56:51,763 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:51,763 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:56:51,763 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:51,764 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:51,764 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:56:51,764 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:51,765 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fcd695f713af429a8b5dbbaf7d9adb01, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=35.9 K 2024-11-16T05:56:51,765 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fcd695f713af429a8b5dbbaf7d9adb01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736606810 2024-11-16T05:56:51,765 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:51,765 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 438ad39633c54cf685bc83fc8ee80871, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731736607961 2024-11-16T05:56:51,766 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c00d0725536a4d1099fd4dceaf6a5517, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610186 2024-11-16T05:56:51,767 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/218ce06330e04eaebd832af44d42483c is 175, key is test_row_0/A:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742015_1191 (size=31549) 2024-11-16T05:56:51,777 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:51,777 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b14944975eaa45d08cb244e8ba82dd09 is 50, key is test_row_0/C:col10/1731736610186/Put/seqid=0 2024-11-16T05:56:51,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742016_1192 (size=12595) 2024-11-16T05:56:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736671823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736671823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736671862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736671862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736671863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:51,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:51,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f5cd629ab1234eeaab7fa6fecf600b87_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5cd629ab1234eeaab7fa6fecf600b87_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:51,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/f623b234cd124f2a88ddf3c838518a0d, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:51,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/f623b234cd124f2a88ddf3c838518a0d is 175, key is test_row_0/A:col10/1731736611464/Put/seqid=0 2024-11-16T05:56:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742017_1193 (size=39749) 2024-11-16T05:56:51,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/f623b234cd124f2a88ddf3c838518a0d 2024-11-16T05:56:51,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f302ede967f94f59819c2a3c3ff8b559 is 50, key is test_row_0/B:col10/1731736611464/Put/seqid=0 2024-11-16T05:56:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742018_1194 (size=12151) 2024-11-16T05:56:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736672133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736672133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T05:56:52,143 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:52,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T05:56:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:52,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:52,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:52,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:52,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736672172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736672172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736672173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,180 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/218ce06330e04eaebd832af44d42483c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c 2024-11-16T05:56:52,186 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into 218ce06330e04eaebd832af44d42483c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:52,186 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:52,186 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=13, startTime=1731736611324; duration=0sec 2024-11-16T05:56:52,186 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:52,186 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:56:52,194 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b14944975eaa45d08cb244e8ba82dd09 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b14944975eaa45d08cb244e8ba82dd09 2024-11-16T05:56:52,199 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into b14944975eaa45d08cb244e8ba82dd09(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:52,199 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:52,199 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=13, startTime=1731736611324; duration=0sec 2024-11-16T05:56:52,199 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:52,199 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:56:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:52,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:56:52,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:52,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f302ede967f94f59819c2a3c3ff8b559 2024-11-16T05:56:52,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/2ed10edbe129405c8b1de9e0aa58bb12 is 50, key is test_row_0/C:col10/1731736611464/Put/seqid=0 2024-11-16T05:56:52,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742019_1195 (size=12151) 2024-11-16T05:56:52,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:56:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:52,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:56:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736672642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736672642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736672683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736672683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736672683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:52,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/2ed10edbe129405c8b1de9e0aa58bb12 2024-11-16T05:56:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/f623b234cd124f2a88ddf3c838518a0d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d 2024-11-16T05:56:52,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d, entries=200, sequenceid=244, filesize=38.8 K 2024-11-16T05:56:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/f302ede967f94f59819c2a3c3ff8b559 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559 2024-11-16T05:56:52,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:56:52,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:52,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:52,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T05:56:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/2ed10edbe129405c8b1de9e0aa58bb12 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12 2024-11-16T05:56:52,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T05:56:52,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 771e8f3141d9fe1005e8f3db6690cb30 in 1302ms, sequenceid=244, compaction requested=false 2024-11-16T05:56:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:52,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T05:56:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:52,911 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:52,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:56:52,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116edcbc8ad8e794a6c91d92debc2510fca_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736611539/Put/seqid=0 2024-11-16T05:56:52,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742020_1196 (size=12454) 2024-11-16T05:56:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:53,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:53,327 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116edcbc8ad8e794a6c91d92debc2510fca_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116edcbc8ad8e794a6c91d92debc2510fca_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:53,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/b3f53f23887e49a68e2b34e259cb8c6c, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:53,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/b3f53f23887e49a68e2b34e259cb8c6c is 175, key is test_row_0/A:col10/1731736611539/Put/seqid=0 2024-11-16T05:56:53,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742021_1197 (size=31255) 2024-11-16T05:56:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736673677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736673682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,708 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,709 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,709 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,735 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/b3f53f23887e49a68e2b34e259cb8c6c 2024-11-16T05:56:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b is 50, key is test_row_0/B:col10/1731736611539/Put/seqid=0 2024-11-16T05:56:53,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742022_1198 (size=12301) 2024-11-16T05:56:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736673793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736673793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736674003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736674003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b 2024-11-16T05:56:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/06ca3d6fd51e492389f397e0249699e5 is 50, key is test_row_0/C:col10/1731736611539/Put/seqid=0 2024-11-16T05:56:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742023_1199 (size=12301) 2024-11-16T05:56:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736674312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736674312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,558 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/06ca3d6fd51e492389f397e0249699e5 2024-11-16T05:56:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/b3f53f23887e49a68e2b34e259cb8c6c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c 2024-11-16T05:56:54,567 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c, entries=150, sequenceid=281, filesize=30.5 K 2024-11-16T05:56:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b 2024-11-16T05:56:54,571 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T05:56:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/06ca3d6fd51e492389f397e0249699e5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5 2024-11-16T05:56:54,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T05:56:54,577 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 771e8f3141d9fe1005e8f3db6690cb30 in 1666ms, sequenceid=281, compaction requested=true 2024-11-16T05:56:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T05:56:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T05:56:54,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T05:56:54,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4320 sec 2024-11-16T05:56:54,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4360 sec 2024-11-16T05:56:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:54,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:54,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:54,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e473a7901c194b939fdfc8ca4093fb91_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:54,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742024_1200 (size=17534) 2024-11-16T05:56:54,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736674866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736674868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736674972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736674972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736675183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736675183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:55,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e473a7901c194b939fdfc8ca4093fb91_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e473a7901c194b939fdfc8ca4093fb91_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:55,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6947211703df43d492aa812fc4dff4a4, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:55,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6947211703df43d492aa812fc4dff4a4 is 175, key is test_row_0/A:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:55,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742025_1201 (size=48639) 2024-11-16T05:56:55,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6947211703df43d492aa812fc4dff4a4 2024-11-16T05:56:55,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/86b43be395524bb0b9826c73f82ff39f is 50, key is test_row_0/B:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:55,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742026_1202 (size=12301) 2024-11-16T05:56:55,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736675493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736675493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/86b43be395524bb0b9826c73f82ff39f 2024-11-16T05:56:55,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b02dc97a8f2245c0a2884742f1a453bf is 50, key is test_row_0/C:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:55,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742027_1203 (size=12301) 2024-11-16T05:56:55,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736675722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,725 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:55,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736675733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,734 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736675733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:55,736 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736676003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736676003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:56,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b02dc97a8f2245c0a2884742f1a453bf 2024-11-16T05:56:56,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6947211703df43d492aa812fc4dff4a4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4 2024-11-16T05:56:56,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4, entries=250, sequenceid=294, filesize=47.5 K 2024-11-16T05:56:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/86b43be395524bb0b9826c73f82ff39f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f 2024-11-16T05:56:56,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T05:56:56,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/b02dc97a8f2245c0a2884742f1a453bf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf 2024-11-16T05:56:56,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T05:56:56,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 771e8f3141d9fe1005e8f3db6690cb30 in 1280ms, sequenceid=294, compaction requested=true 2024-11-16T05:56:56,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:56,108 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:56:56,108 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:56,109 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:56,109 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:56,109 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:56,109 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:56,109 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:56,109 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:56,109 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c938b634fe5040a09c094b2d9ae5361a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=48.2 K 2024-11-16T05:56:56,109 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=147.6 K 2024-11-16T05:56:56,109 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:56,109 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4] 2024-11-16T05:56:56,110 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c938b634fe5040a09c094b2d9ae5361a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610186 2024-11-16T05:56:56,110 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218ce06330e04eaebd832af44d42483c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610186 2024-11-16T05:56:56,110 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f302ede967f94f59819c2a3c3ff8b559, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736610832 2024-11-16T05:56:56,110 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f623b234cd124f2a88ddf3c838518a0d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736610832 2024-11-16T05:56:56,110 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d2ab1c5f5a4c538b8899aa5bcbdd7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731736611537 2024-11-16T05:56:56,111 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f53f23887e49a68e2b34e259cb8c6c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731736611537 2024-11-16T05:56:56,111 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b43be395524bb0b9826c73f82ff39f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:56:56,111 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6947211703df43d492aa812fc4dff4a4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:56:56,118 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:56,121 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:56,121 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e9a7ad45b4e64cefa0daa89be54f3687_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:56,121 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/d6a08d5643b24310b92c182dd7c49735 is 50, key is test_row_0/B:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:56,124 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e9a7ad45b4e64cefa0daa89be54f3687_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:56,124 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e9a7ad45b4e64cefa0daa89be54f3687_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:56,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742028_1204 (size=12881) 2024-11-16T05:56:56,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742029_1205 (size=4469) 2024-11-16T05:56:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T05:56:56,303 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:56,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T05:56:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:56,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:56,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:56,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:56,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:56,458 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:56,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f20bf605f2c04027b261cb1100954e5f_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736614867/Put/seqid=0 2024-11-16T05:56:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742030_1206 (size=12454) 2024-11-16T05:56:56,536 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/d6a08d5643b24310b92c182dd7c49735 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/d6a08d5643b24310b92c182dd7c49735 2024-11-16T05:56:56,540 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:56,540 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/d2947c31c1a84991b7487acf74a78481 is 175, key is test_row_0/A:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:56,541 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into d6a08d5643b24310b92c182dd7c49735(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:56,541 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:56,541 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=12, startTime=1731736616108; duration=0sec 2024-11-16T05:56:56,541 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:56,541 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:56:56,541 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:56:56,542 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:56:56,542 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:56:56,542 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:56,543 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b14944975eaa45d08cb244e8ba82dd09, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=48.2 K 2024-11-16T05:56:56,543 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b14944975eaa45d08cb244e8ba82dd09, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731736610186 2024-11-16T05:56:56,543 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ed10edbe129405c8b1de9e0aa58bb12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736610832 2024-11-16T05:56:56,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742031_1207 (size=31835) 2024-11-16T05:56:56,544 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ca3d6fd51e492389f397e0249699e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731736611537 2024-11-16T05:56:56,545 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b02dc97a8f2245c0a2884742f1a453bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:56:56,554 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:56,554 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/27a4e650807442949444a2fd4607eeb0 is 50, key is test_row_0/C:col10/1731736613672/Put/seqid=0 2024-11-16T05:56:56,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742032_1208 (size=12881) 2024-11-16T05:56:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:56,876 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f20bf605f2c04027b261cb1100954e5f_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f20bf605f2c04027b261cb1100954e5f_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6c77fffddc634803a1a7c96f4e470f9f, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6c77fffddc634803a1a7c96f4e470f9f is 175, key is test_row_0/A:col10/1731736614867/Put/seqid=0 2024-11-16T05:56:56,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742033_1209 (size=31255) 2024-11-16T05:56:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:56,950 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/d2947c31c1a84991b7487acf74a78481 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481 2024-11-16T05:56:56,954 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into d2947c31c1a84991b7487acf74a78481(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:56,954 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:56,955 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=12, startTime=1731736616107; duration=0sec 2024-11-16T05:56:56,955 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:56,955 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:56:56,963 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/27a4e650807442949444a2fd4607eeb0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/27a4e650807442949444a2fd4607eeb0 2024-11-16T05:56:56,967 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into 27a4e650807442949444a2fd4607eeb0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:56:56,967 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:56,967 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=12, startTime=1731736616108; duration=0sec 2024-11-16T05:56:56,967 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:56,967 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:56:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:57,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:57,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736677036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736677036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736677142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736677143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,249 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:56:57,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6c77fffddc634803a1a7c96f4e470f9f 2024-11-16T05:56:57,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/17312606c955458b84f066f6c6af56c9 is 50, key is test_row_0/B:col10/1731736614867/Put/seqid=0 2024-11-16T05:56:57,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742034_1210 (size=12301) 2024-11-16T05:56:57,310 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/17312606c955458b84f066f6c6af56c9 2024-11-16T05:56:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/ed654a93ae58498792c5f56a8aba8fa2 is 50, key is test_row_0/C:col10/1731736614867/Put/seqid=0 2024-11-16T05:56:57,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742035_1211 (size=12301) 2024-11-16T05:56:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736677353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736677353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736677662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736677662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:57,721 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/ed654a93ae58498792c5f56a8aba8fa2 2024-11-16T05:56:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/6c77fffddc634803a1a7c96f4e470f9f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f 2024-11-16T05:56:57,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f, entries=150, sequenceid=313, filesize=30.5 K 2024-11-16T05:56:57,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/17312606c955458b84f066f6c6af56c9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9 2024-11-16T05:56:57,734 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T05:56:57,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/ed654a93ae58498792c5f56a8aba8fa2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2 2024-11-16T05:56:57,739 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T05:56:57,740 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 771e8f3141d9fe1005e8f3db6690cb30 in 1282ms, sequenceid=313, compaction requested=false 2024-11-16T05:56:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:57,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T05:56:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T05:56:57,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T05:56:57,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4360 sec 2024-11-16T05:56:57,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4410 sec 2024-11-16T05:56:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:58,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:58,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163650d5e9e71c44b4947824ee619c77f8_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736618175/Put/seqid=0 2024-11-16T05:56:58,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742036_1212 (size=17534) 2024-11-16T05:56:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736678208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736678209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736678313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736678323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T05:56:58,453 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:56:58,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:56:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T05:56:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:56:58,455 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:56:58,456 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:56:58,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:56:58,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736678522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736678533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:56:58,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:56:58,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163650d5e9e71c44b4947824ee619c77f8_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163650d5e9e71c44b4947824ee619c77f8_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:56:58,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1b0fc492bb734257a93493463de0f6e6, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:58,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1b0fc492bb734257a93493463de0f6e6 is 175, key is test_row_0/A:col10/1731736618175/Put/seqid=0 2024-11-16T05:56:58,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742037_1213 (size=48639) 2024-11-16T05:56:58,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:56:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736678832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736678843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:58,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:58,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1b0fc492bb734257a93493463de0f6e6 2024-11-16T05:56:59,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/fc884a54a33441489b5c40b0d0967363 is 50, key is test_row_0/B:col10/1731736618175/Put/seqid=0 2024-11-16T05:56:59,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742038_1214 (size=12301) 2024-11-16T05:56:59,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:56:59,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736679342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:59,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736679352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:59,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/fc884a54a33441489b5c40b0d0967363 2024-11-16T05:56:59,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/603535adab784ea4989e78e43ff4c7aa is 50, key is test_row_0/C:col10/1731736618175/Put/seqid=0 2024-11-16T05:56:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742039_1215 (size=12301) 2024-11-16T05:56:59,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:56:59,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731736679742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:59,744 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:59,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731736679753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:59,754 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:59,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:56:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731736679773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:56:59,775 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 771e8f3141d9fe1005e8f3db6690cb30 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:56:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:56:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:56:59,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/603535adab784ea4989e78e43ff4c7aa 2024-11-16T05:56:59,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:56:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:56:59,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1b0fc492bb734257a93493463de0f6e6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6 2024-11-16T05:56:59,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6, entries=250, sequenceid=331, filesize=47.5 K 2024-11-16T05:56:59,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/fc884a54a33441489b5c40b0d0967363 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363 2024-11-16T05:56:59,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T05:56:59,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/603535adab784ea4989e78e43ff4c7aa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa 2024-11-16T05:56:59,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T05:56:59,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 771e8f3141d9fe1005e8f3db6690cb30 in 1684ms, sequenceid=331, compaction requested=true 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:56:59,860 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:56:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:56:59,860 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:56:59,862 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,862 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,862 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=109.1 K 2024-11-16T05:56:59,862 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/d6a08d5643b24310b92c182dd7c49735, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=36.6 K 2024-11-16T05:56:59,862 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6] 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a08d5643b24310b92c182dd7c49735, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:56:59,862 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2947c31c1a84991b7487acf74a78481, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:56:59,863 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 17312606c955458b84f066f6c6af56c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731736614836 2024-11-16T05:56:59,863 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c77fffddc634803a1a7c96f4e470f9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731736614836 2024-11-16T05:56:59,863 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fc884a54a33441489b5c40b0d0967363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617035 2024-11-16T05:56:59,863 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0fc492bb734257a93493463de0f6e6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617026 2024-11-16T05:56:59,867 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:59,869 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:56:59,870 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c0e461f85d7a460482b0215e0bec9dea is 50, key is test_row_0/B:col10/1731736618175/Put/seqid=0 2024-11-16T05:56:59,871 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111611a93f1c1afc482b87035c78869f477a_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:59,872 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111611a93f1c1afc482b87035c78869f477a_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:59,873 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111611a93f1c1afc482b87035c78869f477a_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:56:59,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742041_1217 (size=4469) 2024-11-16T05:56:59,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742040_1216 (size=12983) 2024-11-16T05:56:59,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T05:56:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:56:59,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:56:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:56:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:59,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:56:59,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:59,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:56:59,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:56:59,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116239cec710ec0406ba72c6984e203fd80_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736618208/Put/seqid=0 2024-11-16T05:56:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742042_1218 (size=12454) 2024-11-16T05:57:00,283 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:00,283 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3def34cb1e604891aa3aec8da079a024 is 175, key is test_row_0/A:col10/1731736618175/Put/seqid=0 2024-11-16T05:57:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742043_1219 (size=31937) 2024-11-16T05:57:00,288 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c0e461f85d7a460482b0215e0bec9dea as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0e461f85d7a460482b0215e0bec9dea 2024-11-16T05:57:00,293 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into c0e461f85d7a460482b0215e0bec9dea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:00,293 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:00,293 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=13, startTime=1731736619860; duration=0sec 2024-11-16T05:57:00,293 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:00,294 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:57:00,294 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:00,295 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:00,295 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:57:00,295 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:00,295 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/27a4e650807442949444a2fd4607eeb0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=36.6 K 2024-11-16T05:57:00,296 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 27a4e650807442949444a2fd4607eeb0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731736613672 2024-11-16T05:57:00,296 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ed654a93ae58498792c5f56a8aba8fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731736614836 2024-11-16T05:57:00,296 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 603535adab784ea4989e78e43ff4c7aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617035 2024-11-16T05:57:00,299 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/3def34cb1e604891aa3aec8da079a024 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024 2024-11-16T05:57:00,304 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into 3def34cb1e604891aa3aec8da079a024(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:00,304 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:00,304 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=13, startTime=1731736619860; duration=0sec 2024-11-16T05:57:00,304 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:00,304 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:57:00,304 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:00,305 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/d1b50116aebb4b27840103a675034620 is 50, key is test_row_0/C:col10/1731736618175/Put/seqid=0 2024-11-16T05:57:00,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742044_1220 (size=12983) 2024-11-16T05:57:00,315 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/d1b50116aebb4b27840103a675034620 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d1b50116aebb4b27840103a675034620 2024-11-16T05:57:00,321 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into d1b50116aebb4b27840103a675034620(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:00,322 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:00,322 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=13, startTime=1731736619860; duration=0sec 2024-11-16T05:57:00,322 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:00,322 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:57:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. as already flushing 2024-11-16T05:57:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736680377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736680378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:00,402 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116239cec710ec0406ba72c6984e203fd80_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116239cec710ec0406ba72c6984e203fd80_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/677bba559f0d4b6db2697788bf36d81b, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/677bba559f0d4b6db2697788bf36d81b is 175, key is test_row_0/A:col10/1731736618208/Put/seqid=0 2024-11-16T05:57:00,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742045_1221 (size=31255) 2024-11-16T05:57:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736680483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736680483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:57:00,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736680695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736680695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:00,807 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/677bba559f0d4b6db2697788bf36d81b 2024-11-16T05:57:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/822cc020c4944f6894a3d1ec77a9dd13 is 50, key is test_row_0/B:col10/1731736618208/Put/seqid=0 2024-11-16T05:57:00,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742046_1222 (size=12301) 2024-11-16T05:57:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736681002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736681002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:01,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/822cc020c4944f6894a3d1ec77a9dd13 2024-11-16T05:57:01,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/356394255cde4325b95732431890c644 is 50, key is test_row_0/C:col10/1731736618208/Put/seqid=0 2024-11-16T05:57:01,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742047_1223 (size=12301) 2024-11-16T05:57:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54940 deadline: 1731736681513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54982 deadline: 1731736681513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:01,638 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/356394255cde4325b95732431890c644 2024-11-16T05:57:01,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/677bba559f0d4b6db2697788bf36d81b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b 2024-11-16T05:57:01,647 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b, entries=150, sequenceid=348, filesize=30.5 K 2024-11-16T05:57:01,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/822cc020c4944f6894a3d1ec77a9dd13 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13 2024-11-16T05:57:01,652 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13, entries=150, sequenceid=348, filesize=12.0 K 2024-11-16T05:57:01,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/356394255cde4325b95732431890c644 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644 2024-11-16T05:57:01,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644, entries=150, sequenceid=348, filesize=12.0 K 2024-11-16T05:57:01,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 771e8f3141d9fe1005e8f3db6690cb30 in 1670ms, sequenceid=348, compaction requested=false 2024-11-16T05:57:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T05:57:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T05:57:01,660 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-16T05:57:01,660 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:01,660 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,660 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:01,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T05:57:01,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2030 sec 2024-11-16T05:57:01,661 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-16T05:57:01,661 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:01,661 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,661 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:01,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.2070 sec 2024-11-16T05:57:01,663 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-16T05:57:01,663 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:01,663 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,663 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:01,668 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-11-16T05:57:01,668 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:01,668 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,668 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:01,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:02,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:57:02,539 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-16T05:57:02,539 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-16T05:57:02,539 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:02,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:57:02,539 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:02,539 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:02,539 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:02,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:02,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:57:02,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:02,540 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:02,540 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:02,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:57:02,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:02,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:02,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:02,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ed2511b70ea24228bdef4b6fcb3018cd_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_0/A:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:02,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742048_1224 (size=12454) 2024-11-16T05:57:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T05:57:02,614 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:02,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:02,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ed2511b70ea24228bdef4b6fcb3018cd_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ed2511b70ea24228bdef4b6fcb3018cd_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:02,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/73a5c5d230264af5ad15a9bc500c3b24, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:02,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/73a5c5d230264af5ad15a9bc500c3b24 is 175, key is test_row_0/A:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:02,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742049_1225 (size=31255) 2024-11-16T05:57:03,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/73a5c5d230264af5ad15a9bc500c3b24 2024-11-16T05:57:03,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e91403a5609a499ba37cce8d2369cdce is 50, key is test_row_0/B:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:03,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742050_1226 (size=12301) 2024-11-16T05:57:03,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e91403a5609a499ba37cce8d2369cdce 2024-11-16T05:57:03,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/fb96b11c98e3413aa6414847a347825b is 50, key is test_row_0/C:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:03,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742051_1227 (size=12301) 2024-11-16T05:57:04,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/fb96b11c98e3413aa6414847a347825b 2024-11-16T05:57:04,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/73a5c5d230264af5ad15a9bc500c3b24 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24 2024-11-16T05:57:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24, entries=150, sequenceid=368, filesize=30.5 K 2024-11-16T05:57:04,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/e91403a5609a499ba37cce8d2369cdce as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce 2024-11-16T05:57:04,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T05:57:04,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/fb96b11c98e3413aa6414847a347825b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b 2024-11-16T05:57:04,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T05:57:04,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 771e8f3141d9fe1005e8f3db6690cb30 in 1709ms, sequenceid=368, compaction requested=true 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:04,247 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 771e8f3141d9fe1005e8f3db6690cb30:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:04,247 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/A is initiating minor compaction (all files) 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/B is initiating minor compaction (all files) 2024-11-16T05:57:04,248 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/A in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:04,248 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/B in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:04,248 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=92.2 K 2024-11-16T05:57:04,248 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0e461f85d7a460482b0215e0bec9dea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=36.7 K 2024-11-16T05:57:04,248 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24] 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e461f85d7a460482b0215e0bec9dea, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617035 2024-11-16T05:57:04,248 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3def34cb1e604891aa3aec8da079a024, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617035 2024-11-16T05:57:04,249 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677bba559f0d4b6db2697788bf36d81b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731736618200 2024-11-16T05:57:04,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 822cc020c4944f6894a3d1ec77a9dd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731736618200 2024-11-16T05:57:04,249 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a5c5d230264af5ad15a9bc500c3b24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731736620373 2024-11-16T05:57:04,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e91403a5609a499ba37cce8d2369cdce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731736620373 2024-11-16T05:57:04,257 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:04,257 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c846def2853443bfa772141e63d1d0c2 is 50, key is test_row_0/B:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:04,259 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:04,264 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163aa1d21d2b444fe6b1ab468840e0fb35_771e8f3141d9fe1005e8f3db6690cb30 store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:04,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742052_1228 (size=13085) 2024-11-16T05:57:04,281 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163aa1d21d2b444fe6b1ab468840e0fb35_771e8f3141d9fe1005e8f3db6690cb30, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:04,281 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163aa1d21d2b444fe6b1ab468840e0fb35_771e8f3141d9fe1005e8f3db6690cb30 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:04,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742053_1229 (size=4469) 2024-11-16T05:57:04,677 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/c846def2853443bfa772141e63d1d0c2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c846def2853443bfa772141e63d1d0c2 2024-11-16T05:57:04,685 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/B of 771e8f3141d9fe1005e8f3db6690cb30 into c846def2853443bfa772141e63d1d0c2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:04,685 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:04,685 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/B, priority=13, startTime=1731736624247; duration=0sec 2024-11-16T05:57:04,685 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:04,685 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:B 2024-11-16T05:57:04,685 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:04,687 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:04,687 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:04,687 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 771e8f3141d9fe1005e8f3db6690cb30/C is initiating minor compaction (all files) 2024-11-16T05:57:04,687 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 771e8f3141d9fe1005e8f3db6690cb30/C in TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:04,687 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d1b50116aebb4b27840103a675034620, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp, totalSize=36.7 K 2024-11-16T05:57:04,687 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b50116aebb4b27840103a675034620, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731736617035 2024-11-16T05:57:04,687 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/38196bacf1154a92b9c30723b4909623 is 175, key is test_row_0/A:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:04,688 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 356394255cde4325b95732431890c644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731736618200 2024-11-16T05:57:04,688 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fb96b11c98e3413aa6414847a347825b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731736620373 2024-11-16T05:57:04,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742054_1230 (size=32039) 2024-11-16T05:57:04,699 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 771e8f3141d9fe1005e8f3db6690cb30#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:04,699 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/34d3c4c9224d45b0b580c4d85b703529 is 50, key is test_row_0/C:col10/1731736622535/Put/seqid=0 2024-11-16T05:57:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742055_1231 (size=13085) 2024-11-16T05:57:05,097 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/38196bacf1154a92b9c30723b4909623 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/38196bacf1154a92b9c30723b4909623 2024-11-16T05:57:05,102 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/A of 771e8f3141d9fe1005e8f3db6690cb30 into 38196bacf1154a92b9c30723b4909623(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:05,102 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:05,102 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/A, priority=13, startTime=1731736624247; duration=0sec 2024-11-16T05:57:05,102 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:05,102 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:A 2024-11-16T05:57:05,108 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/34d3c4c9224d45b0b580c4d85b703529 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/34d3c4c9224d45b0b580c4d85b703529 2024-11-16T05:57:05,113 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 771e8f3141d9fe1005e8f3db6690cb30/C of 771e8f3141d9fe1005e8f3db6690cb30 into 34d3c4c9224d45b0b580c4d85b703529(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:05,113 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 771e8f3141d9fe1005e8f3db6690cb30: 2024-11-16T05:57:05,113 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., storeName=771e8f3141d9fe1005e8f3db6690cb30/C, priority=13, startTime=1731736624247; duration=0sec 2024-11-16T05:57:05,113 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:05,113 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 771e8f3141d9fe1005e8f3db6690cb30:C 2024-11-16T05:57:09,805 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-16T05:57:09,805 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:09,805 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,805 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:09,854 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-16T05:57:09,854 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:09,854 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,854 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:09,875 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-16T05:57:09,875 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:09,875 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,875 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3005 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2914 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1268 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3802 rows 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1267 2024-11-16T05:57:09,875 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3798 rows 2024-11-16T05:57:09,876 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:57:09,876 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:57:09,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:09,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:09,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:57:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:09,879 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736629879"}]},"ts":"1731736629879"} 2024-11-16T05:57:09,881 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:57:09,881 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:57:09,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:57:09,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, UNASSIGN}] 2024-11-16T05:57:09,883 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, UNASSIGN 2024-11-16T05:57:09,884 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:09,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, UNASSIGN because future has completed 2024-11-16T05:57:09,886 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:57:09,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:10,040 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:10,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:57:10,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 771e8f3141d9fe1005e8f3db6690cb30, disabling compactions & flushes 2024-11-16T05:57:10,040 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:10,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:10,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. after waiting 0 ms 2024-11-16T05:57:10,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:10,041 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 771e8f3141d9fe1005e8f3db6690cb30 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T05:57:10,041 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=A 2024-11-16T05:57:10,041 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:10,042 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=B 2024-11-16T05:57:10,042 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:10,042 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 771e8f3141d9fe1005e8f3db6690cb30, store=C 2024-11-16T05:57:10,042 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:10,053 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3b147eb13c84506ab149617f1d35379_771e8f3141d9fe1005e8f3db6690cb30 is 50, key is test_row_1/A:col10/1731736629873/Put/seqid=0 2024-11-16T05:57:10,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742056_1232 (size=9914) 2024-11-16T05:57:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:10,458 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:10,466 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3b147eb13c84506ab149617f1d35379_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3b147eb13c84506ab149617f1d35379_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:10,468 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1506b2d7789c4167a794f25c77ec8652, store: [table=TestAcidGuarantees family=A region=771e8f3141d9fe1005e8f3db6690cb30] 2024-11-16T05:57:10,469 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1506b2d7789c4167a794f25c77ec8652 is 175, key is test_row_1/A:col10/1731736629873/Put/seqid=0 2024-11-16T05:57:10,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742057_1233 (size=22561) 2024-11-16T05:57:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:10,876 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1506b2d7789c4167a794f25c77ec8652 2024-11-16T05:57:10,887 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/4d579e12d0fd4f30b5982d2e3dfbf212 is 50, key is test_row_1/B:col10/1731736629873/Put/seqid=0 2024-11-16T05:57:10,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742058_1234 (size=9857) 2024-11-16T05:57:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:11,292 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/4d579e12d0fd4f30b5982d2e3dfbf212 2024-11-16T05:57:11,305 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/d51d03c55ff54a2199a821be6abe89d0 is 50, key is test_row_1/C:col10/1731736629873/Put/seqid=0 2024-11-16T05:57:11,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742059_1235 (size=9857) 2024-11-16T05:57:11,711 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/d51d03c55ff54a2199a821be6abe89d0 2024-11-16T05:57:11,717 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/A/1506b2d7789c4167a794f25c77ec8652 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1506b2d7789c4167a794f25c77ec8652 2024-11-16T05:57:11,722 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1506b2d7789c4167a794f25c77ec8652, entries=100, sequenceid=377, filesize=22.0 K 2024-11-16T05:57:11,723 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/B/4d579e12d0fd4f30b5982d2e3dfbf212 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/4d579e12d0fd4f30b5982d2e3dfbf212 2024-11-16T05:57:11,727 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/4d579e12d0fd4f30b5982d2e3dfbf212, entries=100, sequenceid=377, filesize=9.6 K 2024-11-16T05:57:11,728 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/.tmp/C/d51d03c55ff54a2199a821be6abe89d0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d51d03c55ff54a2199a821be6abe89d0 2024-11-16T05:57:11,733 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d51d03c55ff54a2199a821be6abe89d0, entries=100, sequenceid=377, filesize=9.6 K 2024-11-16T05:57:11,737 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 771e8f3141d9fe1005e8f3db6690cb30 in 1697ms, sequenceid=377, compaction requested=false 2024-11-16T05:57:11,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24] to archive 2024-11-16T05:57:11,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:11,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/794041823bd8458294913d9b705abe66 2024-11-16T05:57:11,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0c3847fd2f19435593788573125a2c52 2024-11-16T05:57:11,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/7675cb7fb17b4d68bdbbf6a91ab49803 2024-11-16T05:57:11,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/eda201e417c74b27ac6fd14475658eff 2024-11-16T05:57:11,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/dca51fe6d7194497bb2e4de11ca711ea 2024-11-16T05:57:11,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3638e1a08bab460a8fc452269ede4eed 2024-11-16T05:57:11,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/59ca5e7efd044614855e93ba9dfcb315 2024-11-16T05:57:11,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ba48f2178ee44a97b5505f1c2f6cf158 2024-11-16T05:57:11,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/ec092435fbb34cdf9a68423b59217d5d 2024-11-16T05:57:11,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/e9347c47d4b347648a000e276700d39e 2024-11-16T05:57:11,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/0b31865bf2de476abc18392d4b5f1b6e 2024-11-16T05:57:11,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/24ba7504ba524e809710d29383d22938 2024-11-16T05:57:11,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/a4bae0798d4a47f594437a8ef5870e89 2024-11-16T05:57:11,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/218ce06330e04eaebd832af44d42483c 2024-11-16T05:57:11,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/f623b234cd124f2a88ddf3c838518a0d 2024-11-16T05:57:11,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/b3f53f23887e49a68e2b34e259cb8c6c 2024-11-16T05:57:11,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6947211703df43d492aa812fc4dff4a4 2024-11-16T05:57:11,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/d2947c31c1a84991b7487acf74a78481 2024-11-16T05:57:11,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/6c77fffddc634803a1a7c96f4e470f9f 2024-11-16T05:57:11,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1b0fc492bb734257a93493463de0f6e6 2024-11-16T05:57:11,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/3def34cb1e604891aa3aec8da079a024 2024-11-16T05:57:11,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/677bba559f0d4b6db2697788bf36d81b 2024-11-16T05:57:11,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/73a5c5d230264af5ad15a9bc500c3b24 2024-11-16T05:57:11,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/8ba9a2a573894959b359e7a900e554c6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/9d04b5039ffa40169f4e5b4ccbdacb0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/be9ff695e6f84493b900f3e92bdd24b8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c938b634fe5040a09c094b2d9ae5361a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/d6a08d5643b24310b92c182dd7c49735, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0e461f85d7a460482b0215e0bec9dea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce] to archive 2024-11-16T05:57:11,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:11,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/cf02cb03feae43679b1a45c21f68524d 2024-11-16T05:57:11,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/36a0eee8ffc14f76a5311c1d7d6d8183 2024-11-16T05:57:11,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/8ba9a2a573894959b359e7a900e554c6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/8ba9a2a573894959b359e7a900e554c6 2024-11-16T05:57:11,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/96b64cc566304624a35ca0c2d3765714 2024-11-16T05:57:11,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/022b0247d866415db32311b169925b2f 2024-11-16T05:57:11,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/9d04b5039ffa40169f4e5b4ccbdacb0c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/9d04b5039ffa40169f4e5b4ccbdacb0c 2024-11-16T05:57:11,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e6d7a458ee9e488980b9c91de8f22f63 2024-11-16T05:57:11,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f74ce894d6d343ad9fd3f74b46dc8429 2024-11-16T05:57:11,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/47d162bf003a4ee192056007be7c85bd 2024-11-16T05:57:11,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/be9ff695e6f84493b900f3e92bdd24b8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/be9ff695e6f84493b900f3e92bdd24b8 2024-11-16T05:57:11,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/1462b56349f64385bfa7f5c1ef98e0a5 2024-11-16T05:57:11,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/0b6d94dcc17b4a1c96f5499416994c13 2024-11-16T05:57:11,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c938b634fe5040a09c094b2d9ae5361a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c938b634fe5040a09c094b2d9ae5361a 2024-11-16T05:57:11,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/018b7acc9c2e4f40a720d4fe766b3627 2024-11-16T05:57:11,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/f302ede967f94f59819c2a3c3ff8b559 2024-11-16T05:57:11,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0d2ab1c5f5a4c538b8899aa5bcbdd7b 2024-11-16T05:57:11,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/d6a08d5643b24310b92c182dd7c49735 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/d6a08d5643b24310b92c182dd7c49735 2024-11-16T05:57:11,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/86b43be395524bb0b9826c73f82ff39f 2024-11-16T05:57:11,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/17312606c955458b84f066f6c6af56c9 2024-11-16T05:57:11,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0e461f85d7a460482b0215e0bec9dea to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c0e461f85d7a460482b0215e0bec9dea 2024-11-16T05:57:11,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/fc884a54a33441489b5c40b0d0967363 2024-11-16T05:57:11,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/822cc020c4944f6894a3d1ec77a9dd13 2024-11-16T05:57:11,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/e91403a5609a499ba37cce8d2369cdce 2024-11-16T05:57:11,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/7dbd27ed998944958648c8ab61648d7e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/8393fe0665124d399c96852fdfbefcb0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fcd695f713af429a8b5dbbaf7d9adb01, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b14944975eaa45d08cb244e8ba82dd09, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/27a4e650807442949444a2fd4607eeb0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d1b50116aebb4b27840103a675034620, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b] to archive 2024-11-16T05:57:11,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:11,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/a993f528b729462281dd7b7be68f7c09 2024-11-16T05:57:11,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b2753840650543479329fa0a20012322 2024-11-16T05:57:11,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/7dbd27ed998944958648c8ab61648d7e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/7dbd27ed998944958648c8ab61648d7e 2024-11-16T05:57:11,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/75dc74d0a03e4fdba6f01edbdc5df52e 2024-11-16T05:57:11,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/71e944360af84da6934ef44050c96da1 2024-11-16T05:57:11,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/8393fe0665124d399c96852fdfbefcb0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/8393fe0665124d399c96852fdfbefcb0 2024-11-16T05:57:11,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/da748a87e179464da37901ec4cc64195 2024-11-16T05:57:11,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/3acd8cac06eb494db49b095bb057f9cf 2024-11-16T05:57:11,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/0fc68f524c3e40a8b01c233f7828b82a 2024-11-16T05:57:11,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fcd695f713af429a8b5dbbaf7d9adb01 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fcd695f713af429a8b5dbbaf7d9adb01 2024-11-16T05:57:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/6cb59dd5fc3348bb8c9eafdf73e192cf 2024-11-16T05:57:11,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/438ad39633c54cf685bc83fc8ee80871 2024-11-16T05:57:11,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b14944975eaa45d08cb244e8ba82dd09 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b14944975eaa45d08cb244e8ba82dd09 2024-11-16T05:57:11,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/c00d0725536a4d1099fd4dceaf6a5517 2024-11-16T05:57:11,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/2ed10edbe129405c8b1de9e0aa58bb12 2024-11-16T05:57:11,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/06ca3d6fd51e492389f397e0249699e5 2024-11-16T05:57:11,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/27a4e650807442949444a2fd4607eeb0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/27a4e650807442949444a2fd4607eeb0 2024-11-16T05:57:11,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/b02dc97a8f2245c0a2884742f1a453bf 2024-11-16T05:57:11,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/ed654a93ae58498792c5f56a8aba8fa2 2024-11-16T05:57:11,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d1b50116aebb4b27840103a675034620 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d1b50116aebb4b27840103a675034620 2024-11-16T05:57:11,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/603535adab784ea4989e78e43ff4c7aa 2024-11-16T05:57:11,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/356394255cde4325b95732431890c644 2024-11-16T05:57:11,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/fb96b11c98e3413aa6414847a347825b 2024-11-16T05:57:11,865 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits/380.seqid, newMaxSeqId=380, maxSeqId=4 2024-11-16T05:57:11,866 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. 2024-11-16T05:57:11,867 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 771e8f3141d9fe1005e8f3db6690cb30: Waiting for close lock at 1731736630040Running coprocessor pre-close hooks at 1731736630040Disabling compacts and flushes for region at 1731736630040Disabling writes for close at 1731736630040Obtaining lock to block concurrent updates at 1731736630041 (+1 ms)Preparing flush snapshotting stores in 771e8f3141d9fe1005e8f3db6690cb30 at 1731736630041Finished memstore snapshotting TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731736630042 (+1 ms)Flushing stores of TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30. at 1731736630043 (+1 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/A: creating writer at 1731736630043Flushing mob file 771e8f3141d9fe1005e8f3db6690cb30/A: appending metadata at 1731736630053 (+10 ms)Flushing mob file 771e8f3141d9fe1005e8f3db6690cb30/A: closing flushed file at 1731736630053Flushing 771e8f3141d9fe1005e8f3db6690cb30/A: appending metadata at 1731736630468 (+415 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/A: closing flushed file at 1731736630469 (+1 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/B: creating writer at 1731736630883 (+414 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/B: appending metadata at 1731736630886 (+3 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/B: closing flushed file at 1731736630886Flushing 771e8f3141d9fe1005e8f3db6690cb30/C: creating writer at 1731736631299 (+413 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/C: appending metadata at 1731736631304 (+5 ms)Flushing 771e8f3141d9fe1005e8f3db6690cb30/C: closing flushed file at 1731736631304Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dc43642: reopening flushed file at 1731736631716 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61553d63: reopening flushed file at 1731736631722 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21122a6: reopening flushed file at 1731736631727 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 771e8f3141d9fe1005e8f3db6690cb30 in 1697ms, sequenceid=377, compaction requested=false at 1731736631737 (+10 ms)Writing region close event to WAL at 1731736631861 (+124 ms)Running coprocessor post-close hooks at 1731736631866 (+5 ms)Closed at 1731736631866 2024-11-16T05:57:11,869 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:11,869 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=771e8f3141d9fe1005e8f3db6690cb30, regionState=CLOSED 2024-11-16T05:57:11,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:11,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T05:57:11,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 771e8f3141d9fe1005e8f3db6690cb30, server=3456ee6a3164,45735,1731736558702 in 1.9860 sec 2024-11-16T05:57:11,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-16T05:57:11,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=771e8f3141d9fe1005e8f3db6690cb30, UNASSIGN in 1.9950 sec 2024-11-16T05:57:11,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T05:57:11,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9990 sec 2024-11-16T05:57:11,884 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736631883"}]},"ts":"1731736631883"} 2024-11-16T05:57:11,886 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:57:11,886 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:57:11,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0100 sec 2024-11-16T05:57:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T05:57:12,034 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:12,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:57:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:57:12,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,050 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,054 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits] 2024-11-16T05:57:12,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1506b2d7789c4167a794f25c77ec8652 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/1506b2d7789c4167a794f25c77ec8652 2024-11-16T05:57:12,065 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/38196bacf1154a92b9c30723b4909623 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/A/38196bacf1154a92b9c30723b4909623 2024-11-16T05:57:12,071 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/4d579e12d0fd4f30b5982d2e3dfbf212 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/4d579e12d0fd4f30b5982d2e3dfbf212 2024-11-16T05:57:12,073 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c846def2853443bfa772141e63d1d0c2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/B/c846def2853443bfa772141e63d1d0c2 2024-11-16T05:57:12,075 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/34d3c4c9224d45b0b580c4d85b703529 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/34d3c4c9224d45b0b580c4d85b703529 2024-11-16T05:57:12,078 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d51d03c55ff54a2199a821be6abe89d0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/C/d51d03c55ff54a2199a821be6abe89d0 2024-11-16T05:57:12,081 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits/380.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30/recovered.edits/380.seqid 2024-11-16T05:57:12,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,082 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:57:12,082 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:57:12,084 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:57:12,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111607120968b4b24163b56aa6f7e49152ff_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111607120968b4b24163b56aa6f7e49152ff_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,101 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116239cec710ec0406ba72c6984e203fd80_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116239cec710ec0406ba72c6984e203fd80_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,103 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163650d5e9e71c44b4947824ee619c77f8_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163650d5e9e71c44b4947824ee619c77f8_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,105 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636abde86cba24e7ab2097790ce4c8bc4_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636abde86cba24e7ab2097790ce4c8bc4_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,106 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163ed48ddfa9144349945d6950681fabfb_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163ed48ddfa9144349945d6950681fabfb_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,108 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b72d9cabe234575afcb0f6756b5f35f_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b72d9cabe234575afcb0f6756b5f35f_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,109 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d2513ee0c2b40938c38b5e583fcbf18_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d2513ee0c2b40938c38b5e583fcbf18_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,110 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165180ccbae6cf42d08214e527f51428b7_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165180ccbae6cf42d08214e527f51428b7_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,112 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c3b876343d74ba79f3dc7b159f99fa9_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c3b876343d74ba79f3dc7b159f99fa9_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,114 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b8ad063a120443f9d736fa33acbf951_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b8ad063a120443f9d736fa33acbf951_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,115 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3b147eb13c84506ab149617f1d35379_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b3b147eb13c84506ab149617f1d35379_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,118 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc638ebd7c0c45d583a3de7d6f1f1dff_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dc638ebd7c0c45d583a3de7d6f1f1dff_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,119 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e473a7901c194b939fdfc8ca4093fb91_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e473a7901c194b939fdfc8ca4093fb91_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,121 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ed2511b70ea24228bdef4b6fcb3018cd_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ed2511b70ea24228bdef4b6fcb3018cd_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,124 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116edcbc8ad8e794a6c91d92debc2510fca_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116edcbc8ad8e794a6c91d92debc2510fca_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,129 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f20bf605f2c04027b261cb1100954e5f_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f20bf605f2c04027b261cb1100954e5f_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,132 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5cd629ab1234eeaab7fa6fecf600b87_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f5cd629ab1234eeaab7fa6fecf600b87_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,134 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcf8efbb96e5416b887fbb7daabbecf6_771e8f3141d9fe1005e8f3db6690cb30 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcf8efbb96e5416b887fbb7daabbecf6_771e8f3141d9fe1005e8f3db6690cb30 2024-11-16T05:57:12,135 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:57:12,139 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:57:12,143 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:57:12,147 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:57:12,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:57:12,149 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736632149"}]},"ts":"9223372036854775807"} 2024-11-16T05:57:12,153 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:57:12,153 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 771e8f3141d9fe1005e8f3db6690cb30, NAME => 'TestAcidGuarantees,,1731736598467.771e8f3141d9fe1005e8f3db6690cb30.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:57:12,153 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:57:12,154 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736632154"}]},"ts":"9223372036854775807"} 2024-11-16T05:57:12,157 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:57:12,159 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 125 msec 2024-11-16T05:57:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T05:57:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:57:12,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:12,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1574838696_22 at /127.0.0.1:39828 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2956 (was 3047) 2024-11-16T05:57:12,370 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=2955 2024-11-16T05:57:12,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:57:12,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:57:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:12,374 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:57:12,374 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:12,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-16T05:57:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-16T05:57:12,374 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:57:12,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742060_1236 (size=1061) 2024-11-16T05:57:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-16T05:57:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-16T05:57:12,783 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:57:12,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742061_1237 (size=53) 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ba572f15df9a669de020b29c7e6acd49, disabling compactions & flushes 2024-11-16T05:57:12,795 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. after waiting 0 ms 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:12,795 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:12,795 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba572f15df9a669de020b29c7e6acd49: Waiting for close lock at 1731736632795Disabling compacts and flushes for region at 1731736632795Disabling writes for close at 1731736632795Writing region close event to WAL at 1731736632795Closed at 1731736632795 2024-11-16T05:57:12,796 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:57:12,796 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736632796"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736632796"}]},"ts":"1731736632796"} 2024-11-16T05:57:12,798 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:57:12,799 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:57:12,799 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736632799"}]},"ts":"1731736632799"} 2024-11-16T05:57:12,801 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:57:12,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, ASSIGN}] 2024-11-16T05:57:12,803 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, ASSIGN 2024-11-16T05:57:12,804 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:57:12,955 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ba572f15df9a669de020b29c7e6acd49, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:12,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, ASSIGN because future has completed 2024-11-16T05:57:12,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-16T05:57:13,112 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:57:13,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:57:13,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,114 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,115 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:13,115 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba572f15df9a669de020b29c7e6acd49 columnFamilyName A 2024-11-16T05:57:13,115 DEBUG [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:13,115 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(327): Store=ba572f15df9a669de020b29c7e6acd49/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:13,115 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,116 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:13,116 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba572f15df9a669de020b29c7e6acd49 columnFamilyName B 2024-11-16T05:57:13,116 DEBUG [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:13,117 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(327): Store=ba572f15df9a669de020b29c7e6acd49/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:13,117 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,118 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:13,118 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba572f15df9a669de020b29c7e6acd49 columnFamilyName C 2024-11-16T05:57:13,118 DEBUG [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:13,118 INFO [StoreOpener-ba572f15df9a669de020b29c7e6acd49-1 {}] regionserver.HStore(327): Store=ba572f15df9a669de020b29c7e6acd49/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:13,118 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,118 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,119 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,119 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,120 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,120 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,121 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:57:13,122 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,123 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:57:13,124 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened ba572f15df9a669de020b29c7e6acd49; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60172374, jitterRate=-0.10336175560951233}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:57:13,124 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,124 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for ba572f15df9a669de020b29c7e6acd49: Running coprocessor pre-open hook at 1731736633112Writing region info on filesystem at 1731736633112Initializing all the Stores at 1731736633113 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736633113Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736633113Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736633113Cleaning up temporary data from old regions at 1731736633120 (+7 ms)Running coprocessor post-open hooks at 1731736633124 (+4 ms)Region opened successfully at 1731736633124 2024-11-16T05:57:13,125 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., pid=55, masterSystemTime=1731736633109 2024-11-16T05:57:13,127 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,127 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ba572f15df9a669de020b29c7e6acd49, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:13,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T05:57:13,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 in 172 msec 2024-11-16T05:57:13,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-16T05:57:13,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, ASSIGN in 329 msec 2024-11-16T05:57:13,133 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:57:13,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736633133"}]},"ts":"1731736633133"} 2024-11-16T05:57:13,135 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:57:13,135 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:57:13,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 763 msec 2024-11-16T05:57:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-16T05:57:13,513 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:13,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1c04bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bab3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:57:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:57:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79a86dc5 2024-11-16T05:57:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:57:13,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:57:13,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,523 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130a82e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,524 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5bd06d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,526 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d02280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,527 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7902b6da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,528 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a2505d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,529 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772d3074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,531 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5189bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,532 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c21a5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,534 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffc8277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,535 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7e5d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,536 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ee1cf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8b4bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,539 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc88ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3184b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,542 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bc44e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,543 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e37429d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,544 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1026d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6cd701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:13,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:13,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4546dc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:13,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:13,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:13,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4beb66dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:13,549 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,549 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,549 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,549 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,549 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,550 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,550 DEBUG [Thread-1111 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,550 DEBUG [Thread-1111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,551 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,552 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,553 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,554 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,556 DEBUG [Thread-1107 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:13,556 DEBUG [Thread-1107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-16T05:57:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,560 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:57:13,561 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:13,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:13,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:13,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9f7f8c0964c742f480338eab903a3610 is 50, key is test_row_0/A:col10/1731736633558/Put/seqid=0 2024-11-16T05:57:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736693621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736633607, max timestamp=1731736633624Num uniques 859; 2024-11-16T05:57:13,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:13,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736633607, max timestamp=1731736633624Num uniques 859; 2024-11-16T05:57:13,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736633607, max timestamp=1731736633624Num uniques 859; 2024-11-16T05:57:13,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742062_1238 (size=14341) 2024-11-16T05:57:13,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736693648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736693649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736693651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736693653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:13,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736693732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736693763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736693763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736693763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736693764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:13,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:13,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736693942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736693977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736693977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736693978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736693978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:14,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:14,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9f7f8c0964c742f480338eab903a3610 2024-11-16T05:57:14,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4aaf246a61ea42c8827995aaf0649719 is 50, key is test_row_0/B:col10/1731736633558/Put/seqid=0 2024-11-16T05:57:14,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742063_1239 (size=12001) 2024-11-16T05:57:14,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4aaf246a61ea42c8827995aaf0649719 2024-11-16T05:57:14,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/99187437a64b4e949472e7a7def47e4c is 50, key is test_row_0/C:col10/1731736633558/Put/seqid=0 2024-11-16T05:57:14,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742064_1240 (size=12001) 2024-11-16T05:57:14,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:14,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:14,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736694253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736694282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736694293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736694293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736694293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:14,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:14,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/99187437a64b4e949472e7a7def47e4c 2024-11-16T05:57:14,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9f7f8c0964c742f480338eab903a3610 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610 2024-11-16T05:57:14,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610, entries=200, sequenceid=12, filesize=14.0 K 2024-11-16T05:57:14,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4aaf246a61ea42c8827995aaf0649719 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719 2024-11-16T05:57:14,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T05:57:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/99187437a64b4e949472e7a7def47e4c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c 2024-11-16T05:57:14,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T05:57:14,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ba572f15df9a669de020b29c7e6acd49 in 928ms, sequenceid=12, compaction requested=false 2024-11-16T05:57:14,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:14,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T05:57:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:14,637 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T05:57:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6309fe72fa064648a88b297f324bd92b is 50, key is test_row_0/A:col10/1731736633649/Put/seqid=0 2024-11-16T05:57:14,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742065_1241 (size=12001) 2024-11-16T05:57:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736694798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736694799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736694802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736694803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736694803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736694913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736694913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,060 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6309fe72fa064648a88b297f324bd92b 2024-11-16T05:57:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4d4f4f77e555434881edefc77d33d040 is 50, key is test_row_0/B:col10/1731736633649/Put/seqid=0 2024-11-16T05:57:15,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742066_1242 (size=12001) 2024-11-16T05:57:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736695123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736695123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736695433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736695433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,494 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4d4f4f77e555434881edefc77d33d040 2024-11-16T05:57:15,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a62155599eb04390925bf8dccf96d6b4 is 50, key is test_row_0/C:col10/1731736633649/Put/seqid=0 2024-11-16T05:57:15,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742067_1243 (size=12001) 2024-11-16T05:57:15,586 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a62155599eb04390925bf8dccf96d6b4 2024-11-16T05:57:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6309fe72fa064648a88b297f324bd92b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b 2024-11-16T05:57:15,603 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:57:15,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/4d4f4f77e555434881edefc77d33d040 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040 2024-11-16T05:57:15,607 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:57:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a62155599eb04390925bf8dccf96d6b4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4 2024-11-16T05:57:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:57:15,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ba572f15df9a669de020b29c7e6acd49 in 977ms, sequenceid=46, compaction requested=false 2024-11-16T05:57:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-16T05:57:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-16T05:57:15,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T05:57:15,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-11-16T05:57:15,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0610 sec 2024-11-16T05:57:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T05:57:15,703 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:15,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-16T05:57:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:15,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:15,706 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:15,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:15,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:15,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b1266c17085f4dd9a80fc5fab46fdbcf is 50, key is test_row_0/A:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:15,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742068_1244 (size=14341) 2024-11-16T05:57:15,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736695848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736695849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736695853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:15,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:15,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:15,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:15,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:15,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736695943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736695943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736695963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736695963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736695964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:16,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736696174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736696174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736696174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b1266c17085f4dd9a80fc5fab46fdbcf 2024-11-16T05:57:16,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e5ca9e771f404e13b25523eecec5ce2d is 50, key is test_row_0/B:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:16,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742069_1245 (size=12001) 2024-11-16T05:57:16,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:16,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736696483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736696483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736696483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:16,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e5ca9e771f404e13b25523eecec5ce2d 2024-11-16T05:57:16,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/5c1087f24eb94402b0d46ed69a9253b2 is 50, key is test_row_0/C:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:16,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742070_1246 (size=12001) 2024-11-16T05:57:16,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/5c1087f24eb94402b0d46ed69a9253b2 2024-11-16T05:57:16,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b1266c17085f4dd9a80fc5fab46fdbcf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf 2024-11-16T05:57:16,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf, entries=200, sequenceid=58, filesize=14.0 K 2024-11-16T05:57:16,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e5ca9e771f404e13b25523eecec5ce2d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d 2024-11-16T05:57:16,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T05:57:16,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/5c1087f24eb94402b0d46ed69a9253b2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2 2024-11-16T05:57:16,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T05:57:16,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ba572f15df9a669de020b29c7e6acd49 in 893ms, sequenceid=58, compaction requested=true 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:16,709 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:16,709 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:16,710 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:16,710 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:16,710 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:16,710 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:16,710 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,710 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,710 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.2 K 2024-11-16T05:57:16,710 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=39.7 K 2024-11-16T05:57:16,711 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7f8c0964c742f480338eab903a3610, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736633558 2024-11-16T05:57:16,711 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aaf246a61ea42c8827995aaf0649719, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736633558 2024-11-16T05:57:16,711 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6309fe72fa064648a88b297f324bd92b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736633647 2024-11-16T05:57:16,711 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4f4f77e555434881edefc77d33d040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736633647 2024-11-16T05:57:16,711 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1266c17085f4dd9a80fc5fab46fdbcf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634785 2024-11-16T05:57:16,712 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ca9e771f404e13b25523eecec5ce2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634791 2024-11-16T05:57:16,722 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:16,723 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6e7003e771494b179e462e46b016f255 is 50, key is test_row_0/B:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:16,723 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:16,724 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6a5af520a526410f8f3b36970a693339 is 50, key is test_row_0/A:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:16,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742072_1248 (size=12104) 2024-11-16T05:57:16,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742071_1247 (size=12104) 2024-11-16T05:57:16,747 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6e7003e771494b179e462e46b016f255 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e7003e771494b179e462e46b016f255 2024-11-16T05:57:16,751 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 6e7003e771494b179e462e46b016f255(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:16,751 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:16,751 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736636709; duration=0sec 2024-11-16T05:57:16,751 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:16,751 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:16,752 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:16,752 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:16,753 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:16,753 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,753 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.2 K 2024-11-16T05:57:16,753 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 99187437a64b4e949472e7a7def47e4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731736633558 2024-11-16T05:57:16,753 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a62155599eb04390925bf8dccf96d6b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736633647 2024-11-16T05:57:16,754 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1087f24eb94402b0d46ed69a9253b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634791 2024-11-16T05:57:16,772 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:16,772 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e5bd0d800ad54097a56e5e3a2b588d0b is 50, key is test_row_0/C:col10/1731736635814/Put/seqid=0 2024-11-16T05:57:16,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-16T05:57:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:16,787 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:16,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742073_1249 (size=12104) 2024-11-16T05:57:16,797 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e5bd0d800ad54097a56e5e3a2b588d0b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e5bd0d800ad54097a56e5e3a2b588d0b 2024-11-16T05:57:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5740684e6c754a7183429acc8a8970ea is 50, key is test_row_0/A:col10/1731736635852/Put/seqid=0 2024-11-16T05:57:16,809 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into e5bd0d800ad54097a56e5e3a2b588d0b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:16,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:16,809 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736636709; duration=0sec 2024-11-16T05:57:16,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:16,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:16,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742074_1250 (size=12001) 2024-11-16T05:57:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:16,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:16,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736696973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736696978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736696993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736696993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:16,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736696993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736697082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736697093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6a5af520a526410f8f3b36970a693339 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6a5af520a526410f8f3b36970a693339 2024-11-16T05:57:17,146 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 6a5af520a526410f8f3b36970a693339(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:17,146 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:17,146 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736636709; duration=0sec 2024-11-16T05:57:17,146 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:17,146 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:17,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5740684e6c754a7183429acc8a8970ea 2024-11-16T05:57:17,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/524cb0b49ac04420a1cb610b3d6cc1a5 is 50, key is test_row_0/B:col10/1731736635852/Put/seqid=0 2024-11-16T05:57:17,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742075_1251 (size=12001) 2024-11-16T05:57:17,261 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/524cb0b49ac04420a1cb610b3d6cc1a5 2024-11-16T05:57:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c21141eedad34743a6ee32fe631793d0 is 50, key is test_row_0/C:col10/1731736635852/Put/seqid=0 2024-11-16T05:57:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736697293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736697303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742076_1252 (size=12001) 2024-11-16T05:57:17,420 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:57:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736697603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736697612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:17,743 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c21141eedad34743a6ee32fe631793d0 2024-11-16T05:57:17,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5740684e6c754a7183429acc8a8970ea as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea 2024-11-16T05:57:17,756 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T05:57:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/524cb0b49ac04420a1cb610b3d6cc1a5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5 2024-11-16T05:57:17,764 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T05:57:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c21141eedad34743a6ee32fe631793d0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0 2024-11-16T05:57:17,789 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T05:57:17,791 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ba572f15df9a669de020b29c7e6acd49 in 1005ms, sequenceid=79, compaction requested=false 2024-11-16T05:57:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-16T05:57:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-16T05:57:17,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T05:57:17,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-16T05:57:17,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.0930 sec 2024-11-16T05:57:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T05:57:17,853 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:17,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-16T05:57:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T05:57:17,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:17,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:17,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T05:57:18,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:57:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/241f8bf18071452b96c59ecee537f04d is 50, key is test_row_0/A:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T05:57:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742077_1253 (size=14341) 2024-11-16T05:57:18,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/241f8bf18071452b96c59ecee537f04d 2024-11-16T05:57:18,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/484f1fd01b3846f8808d55232bbd2772 is 50, key is test_row_0/B:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742078_1254 (size=12001) 2024-11-16T05:57:18,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/484f1fd01b3846f8808d55232bbd2772 2024-11-16T05:57:18,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a35c36fbf7e44d9caf205fdb36c03786 is 50, key is test_row_0/C:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736698113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736698123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742079_1255 (size=12001) 2024-11-16T05:57:18,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a35c36fbf7e44d9caf205fdb36c03786 2024-11-16T05:57:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/241f8bf18071452b96c59ecee537f04d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d 2024-11-16T05:57:18,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T05:57:18,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d, entries=200, sequenceid=96, filesize=14.0 K 2024-11-16T05:57:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/484f1fd01b3846f8808d55232bbd2772 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772 2024-11-16T05:57:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:18,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T05:57:18,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a35c36fbf7e44d9caf205fdb36c03786 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786 2024-11-16T05:57:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T05:57:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T05:57:18,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for ba572f15df9a669de020b29c7e6acd49 in 186ms, sequenceid=96, compaction requested=true 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:18,193 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:18,193 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:18,195 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:18,195 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:18,195 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:18,195 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:18,195 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,195 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,195 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e7003e771494b179e462e46b016f255, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.3 K 2024-11-16T05:57:18,195 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6a5af520a526410f8f3b36970a693339, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.5 K 2024-11-16T05:57:18,197 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e7003e771494b179e462e46b016f255, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634791 2024-11-16T05:57:18,197 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5af520a526410f8f3b36970a693339, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634791 2024-11-16T05:57:18,197 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5740684e6c754a7183429acc8a8970ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731736635842 2024-11-16T05:57:18,197 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 524cb0b49ac04420a1cb610b3d6cc1a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731736635842 2024-11-16T05:57:18,197 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 484f1fd01b3846f8808d55232bbd2772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736638004 2024-11-16T05:57:18,198 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 241f8bf18071452b96c59ecee537f04d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736636974 2024-11-16T05:57:18,215 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:18,215 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/85a7c6f695804bdca5734a722061b744 is 50, key is test_row_0/A:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,226 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:18,227 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/2e1d24b24dd1429a8d510a0ea7d71798 is 50, key is test_row_0/B:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742080_1256 (size=12207) 2024-11-16T05:57:18,289 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/85a7c6f695804bdca5734a722061b744 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/85a7c6f695804bdca5734a722061b744 2024-11-16T05:57:18,298 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 85a7c6f695804bdca5734a722061b744(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:18,298 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:18,298 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736638193; duration=0sec 2024-11-16T05:57:18,298 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:18,298 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:18,298 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:18,299 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:18,300 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:18,300 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,300 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e5bd0d800ad54097a56e5e3a2b588d0b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.3 K 2024-11-16T05:57:18,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742081_1257 (size=12207) 2024-11-16T05:57:18,300 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5bd0d800ad54097a56e5e3a2b588d0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731736634791 2024-11-16T05:57:18,301 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c21141eedad34743a6ee32fe631793d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731736635842 2024-11-16T05:57:18,302 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35c36fbf7e44d9caf205fdb36c03786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736638004 2024-11-16T05:57:18,307 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/2e1d24b24dd1429a8d510a0ea7d71798 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2e1d24b24dd1429a8d510a0ea7d71798 2024-11-16T05:57:18,315 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 2e1d24b24dd1429a8d510a0ea7d71798(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:18,315 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:18,315 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736638193; duration=0sec 2024-11-16T05:57:18,315 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:18,315 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:18,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,325 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,328 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:18,328 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/de14d6083d9a4a59ab34d25ae9652926 is 50, key is test_row_0/C:col10/1731736638005/Put/seqid=0 2024-11-16T05:57:18,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/93217eb5a83841edafc886463486e440 is 50, key is test_row_0/A:col10/1731736638050/Put/seqid=0 2024-11-16T05:57:18,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:18,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742082_1258 (size=12207) 2024-11-16T05:57:18,396 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/de14d6083d9a4a59ab34d25ae9652926 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/de14d6083d9a4a59ab34d25ae9652926 2024-11-16T05:57:18,406 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into de14d6083d9a4a59ab34d25ae9652926(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:18,406 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:18,406 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736638193; duration=0sec 2024-11-16T05:57:18,406 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:18,406 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:18,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742083_1259 (size=12001) 2024-11-16T05:57:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,438 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/93217eb5a83841edafc886463486e440 2024-11-16T05:57:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8a0bf9e4c84b4684bbc6f7535a6983f1 is 50, key is test_row_0/B:col10/1731736638050/Put/seqid=0 2024-11-16T05:57:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T05:57:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742084_1260 (size=12001) 2024-11-16T05:57:18,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8a0bf9e4c84b4684bbc6f7535a6983f1 2024-11-16T05:57:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/616d6b1ad3194a06b7d8358d578461aa is 50, key is test_row_0/C:col10/1731736638050/Put/seqid=0 2024-11-16T05:57:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742085_1261 (size=12001) 2024-11-16T05:57:18,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/616d6b1ad3194a06b7d8358d578461aa 2024-11-16T05:57:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/93217eb5a83841edafc886463486e440 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440 2024-11-16T05:57:18,600 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T05:57:18,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8a0bf9e4c84b4684bbc6f7535a6983f1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1 2024-11-16T05:57:18,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T05:57:18,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/616d6b1ad3194a06b7d8358d578461aa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa 2024-11-16T05:57:18,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T05:57:18,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ba572f15df9a669de020b29c7e6acd49 in 307ms, sequenceid=116, compaction requested=false 2024-11-16T05:57:18,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:18,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:18,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-16T05:57:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-16T05:57:18,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T05:57:18,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 770 msec 2024-11-16T05:57:18,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 784 msec 2024-11-16T05:57:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:18,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:18,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e3ed45f4f99a417a966beeb022f64fd5 is 50, key is test_row_0/A:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742086_1262 (size=14441) 2024-11-16T05:57:18,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736698913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736698923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736698923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T05:57:19,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:19,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-16T05:57:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:19,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:19,007 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:19,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:19,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736699123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736699123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,127 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736699132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736699132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736699133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,136 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e3ed45f4f99a417a966beeb022f64fd5 2024-11-16T05:57:19,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0d91bbdc1430438a94e05a91df2f7479 is 50, key is test_row_0/B:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:19,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742087_1263 (size=12101) 2024-11-16T05:57:19,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0d91bbdc1430438a94e05a91df2f7479 2024-11-16T05:57:19,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/b84e059262934c029cb557b0b9fb41cb is 50, key is test_row_0/C:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:19,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742088_1264 (size=12101) 2024-11-16T05:57:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/b84e059262934c029cb557b0b9fb41cb 2024-11-16T05:57:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e3ed45f4f99a417a966beeb022f64fd5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5 2024-11-16T05:57:19,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5, entries=200, sequenceid=132, filesize=14.1 K 2024-11-16T05:57:19,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0d91bbdc1430438a94e05a91df2f7479 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479 2024-11-16T05:57:19,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T05:57:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/b84e059262934c029cb557b0b9fb41cb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb 2024-11-16T05:57:19,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T05:57:19,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ba572f15df9a669de020b29c7e6acd49 in 596ms, sequenceid=132, compaction requested=true 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:19,355 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:19,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:57:19,355 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:19,357 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:19,357 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:19,357 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:19,357 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:19,357 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,357 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,357 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2e1d24b24dd1429a8d510a0ea7d71798, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.5 K 2024-11-16T05:57:19,357 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/85a7c6f695804bdca5734a722061b744, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.7 K 2024-11-16T05:57:19,358 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1d24b24dd1429a8d510a0ea7d71798, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736638004 2024-11-16T05:57:19,358 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a7c6f695804bdca5734a722061b744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736638004 2024-11-16T05:57:19,360 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0bf9e4c84b4684bbc6f7535a6983f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731736638042 2024-11-16T05:57:19,360 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93217eb5a83841edafc886463486e440, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731736638042 2024-11-16T05:57:19,360 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ed45f4f99a417a966beeb022f64fd5, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638411 2024-11-16T05:57:19,360 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d91bbdc1430438a94e05a91df2f7479, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638417 2024-11-16T05:57:19,373 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:19,374 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d5921819cd7748cd9c486359b5252ab9 is 50, key is test_row_0/B:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:19,384 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:19,386 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/8fa070aaf6544b0da299f78facce28e9 is 50, key is test_row_0/A:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:19,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742089_1265 (size=12409) 2024-11-16T05:57:19,418 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d5921819cd7748cd9c486359b5252ab9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d5921819cd7748cd9c486359b5252ab9 2024-11-16T05:57:19,425 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into d5921819cd7748cd9c486359b5252ab9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:19,425 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:19,427 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736639355; duration=0sec 2024-11-16T05:57:19,428 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:19,428 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:19,428 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:19,429 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:19,429 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:19,429 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,430 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/de14d6083d9a4a59ab34d25ae9652926, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=35.5 K 2024-11-16T05:57:19,430 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting de14d6083d9a4a59ab34d25ae9652926, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731736638004 2024-11-16T05:57:19,430 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 616d6b1ad3194a06b7d8358d578461aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731736638042 2024-11-16T05:57:19,430 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b84e059262934c029cb557b0b9fb41cb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638417 2024-11-16T05:57:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742090_1266 (size=12409) 2024-11-16T05:57:19,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:19,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:19,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:19,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:19,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:19,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:19,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:19,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/867dd79d375542d4b67e974435d6ecde is 50, key is test_row_0/A:col10/1731736638798/Put/seqid=0 2024-11-16T05:57:19,451 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:19,452 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/1139ff0597b14d43a17abed7e280d2df is 50, key is test_row_0/C:col10/1731736638756/Put/seqid=0 2024-11-16T05:57:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742091_1267 (size=14541) 2024-11-16T05:57:19,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/867dd79d375542d4b67e974435d6ecde 2024-11-16T05:57:19,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742092_1268 (size=12409) 2024-11-16T05:57:19,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/23a1268bacf94ca69a61eae948241a0a is 50, key is test_row_0/B:col10/1731736638798/Put/seqid=0 2024-11-16T05:57:19,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736699463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736699466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742093_1269 (size=12151) 2024-11-16T05:57:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736699471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/23a1268bacf94ca69a61eae948241a0a 2024-11-16T05:57:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/3349d3ebe0594e67bbaac2f3166c5fbd is 50, key is test_row_0/C:col10/1731736638798/Put/seqid=0 2024-11-16T05:57:19,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742094_1270 (size=12151) 2024-11-16T05:57:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736699572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736699573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736699583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:19,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736699783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736699783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736699794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:19,841 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/8fa070aaf6544b0da299f78facce28e9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/8fa070aaf6544b0da299f78facce28e9 2024-11-16T05:57:19,855 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 8fa070aaf6544b0da299f78facce28e9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:19,855 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:19,855 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736639355; duration=0sec 2024-11-16T05:57:19,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:19,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:19,865 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/1139ff0597b14d43a17abed7e280d2df as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1139ff0597b14d43a17abed7e280d2df 2024-11-16T05:57:19,872 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into 1139ff0597b14d43a17abed7e280d2df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:19,872 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:19,872 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736639355; duration=0sec 2024-11-16T05:57:19,872 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:19,872 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:19,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/3349d3ebe0594e67bbaac2f3166c5fbd 2024-11-16T05:57:19,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/867dd79d375542d4b67e974435d6ecde as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde 2024-11-16T05:57:19,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde, entries=200, sequenceid=151, filesize=14.2 K 2024-11-16T05:57:19,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/23a1268bacf94ca69a61eae948241a0a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a 2024-11-16T05:57:19,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T05:57:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/3349d3ebe0594e67bbaac2f3166c5fbd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd 2024-11-16T05:57:19,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:19,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:19,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:19,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T05:57:19,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ba572f15df9a669de020b29c7e6acd49 in 498ms, sequenceid=151, compaction requested=false 2024-11-16T05:57:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T05:57:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:20,082 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/46f8ffc1a31b42eab65e9e7297ae1401 is 50, key is test_row_0/A:col10/1731736639467/Put/seqid=0 2024-11-16T05:57:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:20,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:20,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742095_1271 (size=9757) 2024-11-16T05:57:20,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/46f8ffc1a31b42eab65e9e7297ae1401 2024-11-16T05:57:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34eb40eadbf34456972c910191c2217f is 50, key is test_row_0/B:col10/1731736639467/Put/seqid=0 2024-11-16T05:57:20,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736700130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736700132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736700134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:20,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742096_1272 (size=9757) 2024-11-16T05:57:20,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34eb40eadbf34456972c910191c2217f 2024-11-16T05:57:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 is 50, key is test_row_0/C:col10/1731736639467/Put/seqid=0 2024-11-16T05:57:20,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742097_1273 (size=9757) 2024-11-16T05:57:20,172 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 2024-11-16T05:57:20,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/46f8ffc1a31b42eab65e9e7297ae1401 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401 2024-11-16T05:57:20,188 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T05:57:20,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34eb40eadbf34456972c910191c2217f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f 2024-11-16T05:57:20,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T05:57:20,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 2024-11-16T05:57:20,202 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T05:57:20,203 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ba572f15df9a669de020b29c7e6acd49 in 121ms, sequenceid=167, compaction requested=true 2024-11-16T05:57:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-16T05:57:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-16T05:57:20,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T05:57:20,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-16T05:57:20,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.2030 sec 2024-11-16T05:57:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:20,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:20,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/f0108ba0bd6240ada82a2e28cc932031 is 50, key is test_row_0/A:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742098_1274 (size=12151) 2024-11-16T05:57:20,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/f0108ba0bd6240ada82a2e28cc932031 2024-11-16T05:57:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736700292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736700292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736700295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/43d9d1d6bfef4ccda0e3b435b9b21e2e is 50, key is test_row_0/B:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742099_1275 (size=12151) 2024-11-16T05:57:20,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/43d9d1d6bfef4ccda0e3b435b9b21e2e 2024-11-16T05:57:20,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c694a4b9709743bab27620b3e0e559d8 is 50, key is test_row_0/C:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742100_1276 (size=12151) 2024-11-16T05:57:20,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c694a4b9709743bab27620b3e0e559d8 2024-11-16T05:57:20,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736700403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736700403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736700404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/f0108ba0bd6240ada82a2e28cc932031 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031 2024-11-16T05:57:20,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T05:57:20,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/43d9d1d6bfef4ccda0e3b435b9b21e2e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e 2024-11-16T05:57:20,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T05:57:20,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c694a4b9709743bab27620b3e0e559d8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8 2024-11-16T05:57:20,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T05:57:20,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ba572f15df9a669de020b29c7e6acd49 in 189ms, sequenceid=186, compaction requested=true 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:20,434 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:20,434 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:20,436 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:20,436 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:20,436 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:20,436 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d5921819cd7748cd9c486359b5252ab9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=45.4 K 2024-11-16T05:57:20,436 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48858 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:20,436 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:20,436 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:20,436 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/8fa070aaf6544b0da299f78facce28e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=47.7 K 2024-11-16T05:57:20,436 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5921819cd7748cd9c486359b5252ab9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638417 2024-11-16T05:57:20,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa070aaf6544b0da299f78facce28e9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638417 2024-11-16T05:57:20,437 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a1268bacf94ca69a61eae948241a0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731736638798 2024-11-16T05:57:20,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 867dd79d375542d4b67e974435d6ecde, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731736638784 2024-11-16T05:57:20,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f8ffc1a31b42eab65e9e7297ae1401, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731736639463 2024-11-16T05:57:20,438 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34eb40eadbf34456972c910191c2217f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731736639463 2024-11-16T05:57:20,438 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f0108ba0bd6240ada82a2e28cc932031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:20,438 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d9d1d6bfef4ccda0e3b435b9b21e2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:20,459 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#234 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:20,459 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/9cf949827e2940a28b0afd800b587840 is 50, key is test_row_0/B:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,473 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:20,474 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/32712daf6a514e8a9449050bb5dffe9f is 50, key is test_row_0/A:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742101_1277 (size=12595) 2024-11-16T05:57:20,533 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/9cf949827e2940a28b0afd800b587840 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/9cf949827e2940a28b0afd800b587840 2024-11-16T05:57:20,539 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 9cf949827e2940a28b0afd800b587840(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:20,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,539 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=12, startTime=1731736640434; duration=0sec 2024-11-16T05:57:20,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:20,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:20,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:20,541 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:20,541 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:20,541 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:20,541 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1139ff0597b14d43a17abed7e280d2df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=45.4 K 2024-11-16T05:57:20,541 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1139ff0597b14d43a17abed7e280d2df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731736638417 2024-11-16T05:57:20,542 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3349d3ebe0594e67bbaac2f3166c5fbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731736638798 2024-11-16T05:57:20,542 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44c47f8efcba4c3cb4b0b36fe57b4fe3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731736639463 2024-11-16T05:57:20,543 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c694a4b9709743bab27620b3e0e559d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:20,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742102_1278 (size=12595) 2024-11-16T05:57:20,551 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/32712daf6a514e8a9449050bb5dffe9f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/32712daf6a514e8a9449050bb5dffe9f 2024-11-16T05:57:20,561 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 32712daf6a514e8a9449050bb5dffe9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:20,561 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,561 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=12, startTime=1731736640434; duration=0sec 2024-11-16T05:57:20,562 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:20,562 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:20,566 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:20,566 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f25eec9db15840bfb7a1b8eda451aaad is 50, key is test_row_0/C:col10/1731736640243/Put/seqid=0 2024-11-16T05:57:20,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742103_1279 (size=12595) 2024-11-16T05:57:20,620 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f25eec9db15840bfb7a1b8eda451aaad as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f25eec9db15840bfb7a1b8eda451aaad 2024-11-16T05:57:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:20,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:20,629 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into f25eec9db15840bfb7a1b8eda451aaad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:20,630 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:20,630 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=12, startTime=1731736640434; duration=0sec 2024-11-16T05:57:20,630 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:20,630 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:20,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/ada198a8f75e444aab6c226c9421918b is 50, key is test_row_0/A:col10/1731736640621/Put/seqid=0 2024-11-16T05:57:20,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742104_1280 (size=14541) 2024-11-16T05:57:20,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/ada198a8f75e444aab6c226c9421918b 2024-11-16T05:57:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736700674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736700674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736700684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/97b144f6845640fe80cee2eaee09e36c is 50, key is test_row_0/B:col10/1731736640621/Put/seqid=0 2024-11-16T05:57:20,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742105_1281 (size=12151) 2024-11-16T05:57:20,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/97b144f6845640fe80cee2eaee09e36c 2024-11-16T05:57:20,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/d7495ca36bcd412ea2de19b124e4c793 is 50, key is test_row_0/C:col10/1731736640621/Put/seqid=0 2024-11-16T05:57:20,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742106_1282 (size=12151) 2024-11-16T05:57:20,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/d7495ca36bcd412ea2de19b124e4c793 2024-11-16T05:57:20,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/ada198a8f75e444aab6c226c9421918b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b 2024-11-16T05:57:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736700793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736700794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736700806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:20,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b, entries=200, sequenceid=203, filesize=14.2 K 2024-11-16T05:57:20,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/97b144f6845640fe80cee2eaee09e36c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c 2024-11-16T05:57:20,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T05:57:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/d7495ca36bcd412ea2de19b124e4c793 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793 2024-11-16T05:57:20,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T05:57:20,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ba572f15df9a669de020b29c7e6acd49 in 213ms, sequenceid=203, compaction requested=false 2024-11-16T05:57:20,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:21,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:21,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:21,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:21,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:21,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/3f9eb034eecc4a1fa2defc8050f8e30c is 50, key is test_row_0/A:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736701031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736701033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742107_1283 (size=14541) 2024-11-16T05:57:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736701036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736701143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736701143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736701143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,147 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T05:57:21,153 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:21,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T05:57:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:21,156 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:21,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:21,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736701153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736701163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,168 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:21,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T05:57:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:21,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736701353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736701353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736701373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/3f9eb034eecc4a1fa2defc8050f8e30c 2024-11-16T05:57:21,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8f106ac3858d46fe85afc144582ad633 is 50, key is test_row_0/B:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T05:57:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:21,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742108_1284 (size=12151) 2024-11-16T05:57:21,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8f106ac3858d46fe85afc144582ad633 2024-11-16T05:57:21,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/223295b18a4c4c12942edf598372a9b1 is 50, key is test_row_0/C:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742109_1285 (size=12151) 2024-11-16T05:57:21,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/223295b18a4c4c12942edf598372a9b1 2024-11-16T05:57:21,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/3f9eb034eecc4a1fa2defc8050f8e30c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c 2024-11-16T05:57:21,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c, entries=200, sequenceid=222, filesize=14.2 K 2024-11-16T05:57:21,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/8f106ac3858d46fe85afc144582ad633 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633 2024-11-16T05:57:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633, entries=150, sequenceid=222, filesize=11.9 K 2024-11-16T05:57:21,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/223295b18a4c4c12942edf598372a9b1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1 2024-11-16T05:57:21,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1, entries=150, sequenceid=222, filesize=11.9 K 2024-11-16T05:57:21,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ba572f15df9a669de020b29c7e6acd49 in 598ms, sequenceid=222, compaction requested=true 2024-11-16T05:57:21,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:21,605 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:21,605 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:21,606 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:21,606 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:21,606 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:21,606 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:21,606 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,606 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/9cf949827e2940a28b0afd800b587840, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.0 K 2024-11-16T05:57:21,607 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,607 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/32712daf6a514e8a9449050bb5dffe9f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=40.7 K 2024-11-16T05:57:21,608 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf949827e2940a28b0afd800b587840, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:21,609 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32712daf6a514e8a9449050bb5dffe9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:21,610 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b144f6845640fe80cee2eaee09e36c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731736640291 2024-11-16T05:57:21,612 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada198a8f75e444aab6c226c9421918b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731736640291 2024-11-16T05:57:21,612 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f106ac3858d46fe85afc144582ad633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:21,613 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f9eb034eecc4a1fa2defc8050f8e30c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:21,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T05:57:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,617 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/20d73e873ccd4fd89e786238a0c9d323 is 50, key is test_row_0/A:col10/1731736641034/Put/seqid=0 2024-11-16T05:57:21,634 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:21,634 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6f04c2b10b0141b4832f8d2d952a77dc is 50, key is test_row_0/B:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,643 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:21,644 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/438bfb04fbc94c009dad3db9d6884993 is 50, key is test_row_0/A:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742110_1286 (size=12151) 2024-11-16T05:57:21,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:21,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742112_1288 (size=12697) 2024-11-16T05:57:21,706 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/438bfb04fbc94c009dad3db9d6884993 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/438bfb04fbc94c009dad3db9d6884993 2024-11-16T05:57:21,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742111_1287 (size=12697) 2024-11-16T05:57:21,717 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 438bfb04fbc94c009dad3db9d6884993(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:21,718 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:21,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,718 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736641605; duration=0sec 2024-11-16T05:57:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736701709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,718 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:21,718 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:21,718 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,719 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6f04c2b10b0141b4832f8d2d952a77dc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6f04c2b10b0141b4832f8d2d952a77dc 2024-11-16T05:57:21,720 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:21,720 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:21,720 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:21,720 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f25eec9db15840bfb7a1b8eda451aaad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.0 K 2024-11-16T05:57:21,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736701714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,722 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f25eec9db15840bfb7a1b8eda451aaad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731736640126 2024-11-16T05:57:21,723 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7495ca36bcd412ea2de19b124e4c793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731736640291 2024-11-16T05:57:21,724 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223295b18a4c4c12942edf598372a9b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:21,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736701719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,728 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 6f04c2b10b0141b4832f8d2d952a77dc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:21,728 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:21,728 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736641605; duration=0sec 2024-11-16T05:57:21,728 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:21,728 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:21,742 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:21,743 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f68e01b4a0554d9aba18516c5da31c57 is 50, key is test_row_0/C:col10/1731736640669/Put/seqid=0 2024-11-16T05:57:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:21,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742113_1289 (size=12697) 2024-11-16T05:57:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736701823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736701833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736701833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736702032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736702043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736702043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,064 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/20d73e873ccd4fd89e786238a0c9d323 2024-11-16T05:57:22,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/2280d763b4e648f68d972d4aa433a2cc is 50, key is test_row_0/B:col10/1731736641034/Put/seqid=0 2024-11-16T05:57:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742114_1290 (size=12151) 2024-11-16T05:57:22,209 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f68e01b4a0554d9aba18516c5da31c57 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f68e01b4a0554d9aba18516c5da31c57 2024-11-16T05:57:22,213 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into f68e01b4a0554d9aba18516c5da31c57(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:22,213 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:22,213 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736641605; duration=0sec 2024-11-16T05:57:22,213 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:22,213 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736702343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736702353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736702354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,502 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/2280d763b4e648f68d972d4aa433a2cc 2024-11-16T05:57:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c242d7f685c7484c81aeb4a3b1542402 is 50, key is test_row_0/C:col10/1731736641034/Put/seqid=0 2024-11-16T05:57:22,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742115_1291 (size=12151) 2024-11-16T05:57:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736702853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736702862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736702873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:22,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c242d7f685c7484c81aeb4a3b1542402 2024-11-16T05:57:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/20d73e873ccd4fd89e786238a0c9d323 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323 2024-11-16T05:57:22,953 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T05:57:22,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/2280d763b4e648f68d972d4aa433a2cc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc 2024-11-16T05:57:22,962 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T05:57:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/c242d7f685c7484c81aeb4a3b1542402 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402 2024-11-16T05:57:22,969 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T05:57:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ba572f15df9a669de020b29c7e6acd49 in 1354ms, sequenceid=236, compaction requested=false 2024-11-16T05:57:22,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T05:57:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T05:57:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T05:57:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8170 sec 2024-11-16T05:57:22,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8250 sec 2024-11-16T05:57:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T05:57:23,303 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:23,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T05:57:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:57:23,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:23,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:23,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:57:23,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T05:57:23,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:23,461 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:23,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b2d9cc2b674241f285493ad0354ca4d2 is 50, key is test_row_0/A:col10/1731736641716/Put/seqid=0 2024-11-16T05:57:23,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742116_1292 (size=12151) 2024-11-16T05:57:23,485 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b2d9cc2b674241f285493ad0354ca4d2 2024-11-16T05:57:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e60e3692063d4b22b567c04c9e98862f is 50, key is test_row_0/B:col10/1731736641716/Put/seqid=0 2024-11-16T05:57:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742117_1293 (size=12151) 2024-11-16T05:57:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:57:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:23,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:23,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736703898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736703899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:23,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736703900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:57:23,935 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e60e3692063d4b22b567c04c9e98862f 2024-11-16T05:57:23,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/433d50efbcc248f0a39f722f72e05609 is 50, key is test_row_0/C:col10/1731736641716/Put/seqid=0 2024-11-16T05:57:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742118_1294 (size=12151) 2024-11-16T05:57:23,961 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/433d50efbcc248f0a39f722f72e05609 2024-11-16T05:57:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/b2d9cc2b674241f285493ad0354ca4d2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2 2024-11-16T05:57:23,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T05:57:23,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/e60e3692063d4b22b567c04c9e98862f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f 2024-11-16T05:57:23,976 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T05:57:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/433d50efbcc248f0a39f722f72e05609 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609 2024-11-16T05:57:23,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T05:57:23,984 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ba572f15df9a669de020b29c7e6acd49 in 522ms, sequenceid=257, compaction requested=true 2024-11-16T05:57:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T05:57:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T05:57:23,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T05:57:23,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 678 msec 2024-11-16T05:57:23,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 684 msec 2024-11-16T05:57:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:24,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/edb68e08d4a94c368d313ba4bdc0661a is 50, key is test_row_0/A:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742119_1295 (size=14741) 2024-11-16T05:57:24,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/edb68e08d4a94c368d313ba4bdc0661a 2024-11-16T05:57:24,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/607849ecd78e4eb9b93e3a22030145dd is 50, key is test_row_0/B:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742120_1296 (size=12301) 2024-11-16T05:57:24,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/607849ecd78e4eb9b93e3a22030145dd 2024-11-16T05:57:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736704045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736704046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736704046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/38c9d8e82ec9457987a81ee6e473df0b is 50, key is test_row_0/C:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742121_1297 (size=12301) 2024-11-16T05:57:24,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/38c9d8e82ec9457987a81ee6e473df0b 2024-11-16T05:57:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/edb68e08d4a94c368d313ba4bdc0661a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a 2024-11-16T05:57:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a, entries=200, sequenceid=272, filesize=14.4 K 2024-11-16T05:57:24,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/607849ecd78e4eb9b93e3a22030145dd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd 2024-11-16T05:57:24,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:57:24,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/38c9d8e82ec9457987a81ee6e473df0b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b 2024-11-16T05:57:24,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T05:57:24,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ba572f15df9a669de020b29c7e6acd49 in 88ms, sequenceid=272, compaction requested=true 2024-11-16T05:57:24,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:24,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:24,096 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:24,096 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:24,097 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:24,097 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:24,097 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,097 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6f04c2b10b0141b4832f8d2d952a77dc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=48.1 K 2024-11-16T05:57:24,098 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f04c2b10b0141b4832f8d2d952a77dc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:24,098 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51740 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:24,098 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:24,098 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,098 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/438bfb04fbc94c009dad3db9d6884993, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=50.5 K 2024-11-16T05:57:24,099 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438bfb04fbc94c009dad3db9d6884993, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:24,099 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2280d763b4e648f68d972d4aa433a2cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736641027 2024-11-16T05:57:24,099 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d73e873ccd4fd89e786238a0c9d323, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736641027 2024-11-16T05:57:24,100 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e60e3692063d4b22b567c04c9e98862f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731736641696 2024-11-16T05:57:24,100 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d9cc2b674241f285493ad0354ca4d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731736641696 2024-11-16T05:57:24,100 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 607849ecd78e4eb9b93e3a22030145dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:24,100 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb68e08d4a94c368d313ba4bdc0661a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:24,117 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:24,117 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d3fb7a547cb348b1aeffc8151b33a679 is 50, key is test_row_0/B:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,118 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#255 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:24,119 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/fdaf8f9f1f69485ba6a54d4d0252a66b is 50, key is test_row_0/A:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742122_1298 (size=12983) 2024-11-16T05:57:24,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742123_1299 (size=12983) 2024-11-16T05:57:24,161 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/fdaf8f9f1f69485ba6a54d4d0252a66b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/fdaf8f9f1f69485ba6a54d4d0252a66b 2024-11-16T05:57:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:24,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:24,171 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into fdaf8f9f1f69485ba6a54d4d0252a66b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:24,171 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:24,171 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=12, startTime=1731736644095; duration=0sec 2024-11-16T05:57:24,171 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:24,171 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:24,171 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:24,173 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:24,173 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:24,173 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,173 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f68e01b4a0554d9aba18516c5da31c57, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=48.1 K 2024-11-16T05:57:24,173 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f68e01b4a0554d9aba18516c5da31c57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731736640669 2024-11-16T05:57:24,174 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c242d7f685c7484c81aeb4a3b1542402, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736641027 2024-11-16T05:57:24,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e8f9a16406b64aebadfdb97151cdf47e is 50, key is test_row_0/A:col10/1731736644043/Put/seqid=0 2024-11-16T05:57:24,175 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 433d50efbcc248f0a39f722f72e05609, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731736641696 2024-11-16T05:57:24,177 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c9d8e82ec9457987a81ee6e473df0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742124_1300 (size=14741) 2024-11-16T05:57:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736704197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e8f9a16406b64aebadfdb97151cdf47e 2024-11-16T05:57:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736704203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736704203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,212 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:24,212 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/73bcf19796f049dbaaff60b615bf13ef is 50, key is test_row_0/C:col10/1731736644005/Put/seqid=0 2024-11-16T05:57:24,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/afc1a658f1c145109c95e4ac0094b1db is 50, key is test_row_0/B:col10/1731736644043/Put/seqid=0 2024-11-16T05:57:24,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742125_1301 (size=12983) 2024-11-16T05:57:24,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742126_1302 (size=12301) 2024-11-16T05:57:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736704313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736704313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736704313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T05:57:24,443 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:24,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T05:57:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:24,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:24,450 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:24,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736704523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736704523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736704524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,545 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d3fb7a547cb348b1aeffc8151b33a679 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d3fb7a547cb348b1aeffc8151b33a679 2024-11-16T05:57:24,552 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into d3fb7a547cb348b1aeffc8151b33a679(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:24,552 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:24,552 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=12, startTime=1731736644096; duration=0sec 2024-11-16T05:57:24,552 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:24,552 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:24,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/afc1a658f1c145109c95e4ac0094b1db 2024-11-16T05:57:24,642 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/73bcf19796f049dbaaff60b615bf13ef as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73bcf19796f049dbaaff60b615bf13ef 2024-11-16T05:57:24,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/2dccc441c7184f38909fd23fc16c241e is 50, key is test_row_0/C:col10/1731736644043/Put/seqid=0 2024-11-16T05:57:24,649 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into 73bcf19796f049dbaaff60b615bf13ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:24,649 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:24,649 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=12, startTime=1731736644096; duration=0sec 2024-11-16T05:57:24,649 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:24,649 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:24,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742127_1303 (size=12301) 2024-11-16T05:57:24,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736704833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736704833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736704833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:24,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:24,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/2dccc441c7184f38909fd23fc16c241e 2024-11-16T05:57:25,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/e8f9a16406b64aebadfdb97151cdf47e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e 2024-11-16T05:57:25,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e, entries=200, sequenceid=290, filesize=14.4 K 2024-11-16T05:57:25,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/afc1a658f1c145109c95e4ac0094b1db as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db 2024-11-16T05:57:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:25,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T05:57:25,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/2dccc441c7184f38909fd23fc16c241e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e 2024-11-16T05:57:25,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T05:57:25,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for ba572f15df9a669de020b29c7e6acd49 in 920ms, sequenceid=290, compaction requested=false 2024-11-16T05:57:25,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:25,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:25,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/a425a7c9288a438395dffb3604cded6d is 50, key is test_row_0/A:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742128_1304 (size=12301) 2024-11-16T05:57:25,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736705243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736705248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736705343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736705343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736705343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736705352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736705362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736705563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736705573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:25,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/a425a7c9288a438395dffb3604cded6d 2024-11-16T05:57:25,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/446ed88f8c7c42a7921ccc9bda7de455 is 50, key is test_row_0/B:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:25,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742129_1305 (size=12301) 2024-11-16T05:57:25,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736705873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736705884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:25,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:25,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/446ed88f8c7c42a7921ccc9bda7de455 2024-11-16T05:57:26,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a7e29928a3b44034bf3865f3dfb452fd is 50, key is test_row_0/C:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:26,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742130_1306 (size=12301) 2024-11-16T05:57:26,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736706353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:26,356 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736706363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736706363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:26,367 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,367 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:26,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736706382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:26,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736706403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:26,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:26,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a7e29928a3b44034bf3865f3dfb452fd 2024-11-16T05:57:26,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/a425a7c9288a438395dffb3604cded6d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d 2024-11-16T05:57:26,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-16T05:57:26,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/446ed88f8c7c42a7921ccc9bda7de455 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455 2024-11-16T05:57:26,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455, entries=150, sequenceid=309, filesize=12.0 K 2024-11-16T05:57:26,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/a7e29928a3b44034bf3865f3dfb452fd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd 2024-11-16T05:57:26,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd, entries=150, sequenceid=309, filesize=12.0 K 2024-11-16T05:57:26,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ba572f15df9a669de020b29c7e6acd49 in 1370ms, sequenceid=309, compaction requested=true 2024-11-16T05:57:26,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:26,536 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:26,536 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:26,537 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:26,537 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:26,537 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:26,537 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:26,537 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,537 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,537 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/fdaf8f9f1f69485ba6a54d4d0252a66b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=39.1 K 2024-11-16T05:57:26,537 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d3fb7a547cb348b1aeffc8151b33a679, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.7 K 2024-11-16T05:57:26,538 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fb7a547cb348b1aeffc8151b33a679, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:26,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdaf8f9f1f69485ba6a54d4d0252a66b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:26,539 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting afc1a658f1c145109c95e4ac0094b1db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731736644033 2024-11-16T05:57:26,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f9a16406b64aebadfdb97151cdf47e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731736644033 2024-11-16T05:57:26,539 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a425a7c9288a438395dffb3604cded6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:26,539 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 446ed88f8c7c42a7921ccc9bda7de455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:26,554 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:26,555 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/33e16b42d0da422fb9f0e3eb1d36734c is 50, key is test_row_0/A:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:26,563 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:26,564 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d501708d78da4bbeb724e2027f0631c7 is 50, key is test_row_0/B:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:26,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742131_1307 (size=13085) 2024-11-16T05:57:26,575 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/33e16b42d0da422fb9f0e3eb1d36734c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/33e16b42d0da422fb9f0e3eb1d36734c 2024-11-16T05:57:26,581 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 33e16b42d0da422fb9f0e3eb1d36734c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:26,581 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:26,581 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736646535; duration=0sec 2024-11-16T05:57:26,581 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:26,581 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:26,581 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:26,582 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:26,582 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:26,583 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,583 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73bcf19796f049dbaaff60b615bf13ef, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.7 K 2024-11-16T05:57:26,583 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73bcf19796f049dbaaff60b615bf13ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731736643895 2024-11-16T05:57:26,583 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dccc441c7184f38909fd23fc16c241e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731736644033 2024-11-16T05:57:26,584 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e29928a3b44034bf3865f3dfb452fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742132_1308 (size=13085) 2024-11-16T05:57:26,600 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#266 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:26,601 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/7fdee599df6b447e9f9e352446562b81 is 50, key is test_row_0/C:col10/1731736644193/Put/seqid=0 2024-11-16T05:57:26,606 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d501708d78da4bbeb724e2027f0631c7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d501708d78da4bbeb724e2027f0631c7 2024-11-16T05:57:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742133_1309 (size=13085) 2024-11-16T05:57:26,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T05:57:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,612 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:26,615 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into d501708d78da4bbeb724e2027f0631c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:26,615 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:26,615 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736646536; duration=0sec 2024-11-16T05:57:26,615 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:26,615 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/65431e8dd06745709298c20b39d85101 is 50, key is test_row_0/A:col10/1731736645228/Put/seqid=0 2024-11-16T05:57:26,623 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/7fdee599df6b447e9f9e352446562b81 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/7fdee599df6b447e9f9e352446562b81 2024-11-16T05:57:26,628 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into 7fdee599df6b447e9f9e352446562b81(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:26,628 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:26,628 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736646536; duration=0sec 2024-11-16T05:57:26,628 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:26,628 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742134_1310 (size=12301) 2024-11-16T05:57:26,634 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/65431e8dd06745709298c20b39d85101 2024-11-16T05:57:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/cf54c5ffd37d4a77846aa404a2364aeb is 50, key is test_row_0/B:col10/1731736645228/Put/seqid=0 2024-11-16T05:57:26,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742135_1311 (size=12301) 2024-11-16T05:57:26,667 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/cf54c5ffd37d4a77846aa404a2364aeb 2024-11-16T05:57:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e4328c910d0640d58148f81757b16202 is 50, key is test_row_0/C:col10/1731736645228/Put/seqid=0 2024-11-16T05:57:26,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742136_1312 (size=12301) 2024-11-16T05:57:26,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e4328c910d0640d58148f81757b16202 2024-11-16T05:57:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/65431e8dd06745709298c20b39d85101 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101 2024-11-16T05:57:26,700 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T05:57:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/cf54c5ffd37d4a77846aa404a2364aeb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb 2024-11-16T05:57:26,709 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T05:57:26,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e4328c910d0640d58148f81757b16202 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202 2024-11-16T05:57:26,716 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T05:57:26,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for ba572f15df9a669de020b29c7e6acd49 in 104ms, sequenceid=326, compaction requested=false 2024-11-16T05:57:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T05:57:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T05:57:26,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T05:57:26,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2690 sec 2024-11-16T05:57:26,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2790 sec 2024-11-16T05:57:27,249 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:57:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:27,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:27,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:27,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6ac918d4538f4aa6926a4d3431bd59ab is 50, key is test_row_0/A:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:27,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742137_1313 (size=12301) 2024-11-16T05:57:27,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6ac918d4538f4aa6926a4d3431bd59ab 2024-11-16T05:57:27,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/a0fd34573feb46dbba00e201323faa4e is 50, key is test_row_0/B:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:27,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742138_1314 (size=12301) 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736647424, max timestamp=1731736647465Num uniques 859; 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736647424, max timestamp=1731736647465Num uniques 859; 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736647424, max timestamp=1731736647465Num uniques 859; 2024-11-16T05:57:27,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736707511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736707518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736707623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736707623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/a0fd34573feb46dbba00e201323faa4e 2024-11-16T05:57:27,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736707833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e89f36c888c04bc8bac2a7d39918f2df is 50, key is test_row_0/C:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736707843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:27,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742139_1315 (size=12301) 2024-11-16T05:57:28,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736708143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736708153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e89f36c888c04bc8bac2a7d39918f2df 2024-11-16T05:57:28,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/6ac918d4538f4aa6926a4d3431bd59ab as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab 2024-11-16T05:57:28,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T05:57:28,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/a0fd34573feb46dbba00e201323faa4e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e 2024-11-16T05:57:28,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T05:57:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e89f36c888c04bc8bac2a7d39918f2df as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df 2024-11-16T05:57:28,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T05:57:28,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ba572f15df9a669de020b29c7e6acd49 in 859ms, sequenceid=338, compaction requested=true 2024-11-16T05:57:28,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:28,270 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:28,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:28,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:28,271 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:28,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:28,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:28,271 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,271 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/33e16b42d0da422fb9f0e3eb1d36734c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.8 K 2024-11-16T05:57:28,271 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:28,272 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:28,272 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,272 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d501708d78da4bbeb724e2027f0631c7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.8 K 2024-11-16T05:57:28,272 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e16b42d0da422fb9f0e3eb1d36734c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:28,272 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d501708d78da4bbeb724e2027f0631c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:28,285 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf54c5ffd37d4a77846aa404a2364aeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731736645228 2024-11-16T05:57:28,286 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 65431e8dd06745709298c20b39d85101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731736645228 2024-11-16T05:57:28,286 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0fd34573feb46dbba00e201323faa4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:28,286 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac918d4538f4aa6926a4d3431bd59ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:28,307 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:28,308 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/be73e2ff283d4b2b8fea74f5ad2ba9b8 is 50, key is test_row_0/A:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:28,319 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:28,320 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/ef87a42643664c2f9b222ec80401e2b6 is 50, key is test_row_0/B:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742140_1316 (size=13187) 2024-11-16T05:57:28,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/be73e2ff283d4b2b8fea74f5ad2ba9b8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/be73e2ff283d4b2b8fea74f5ad2ba9b8 2024-11-16T05:57:28,352 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into be73e2ff283d4b2b8fea74f5ad2ba9b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:28,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:28,352 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736648270; duration=0sec 2024-11-16T05:57:28,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:28,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:28,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:28,354 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:28,354 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:28,354 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,354 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/7fdee599df6b447e9f9e352446562b81, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=36.8 K 2024-11-16T05:57:28,354 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fdee599df6b447e9f9e352446562b81, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731736644193 2024-11-16T05:57:28,355 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e4328c910d0640d58148f81757b16202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731736645228 2024-11-16T05:57:28,356 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e89f36c888c04bc8bac2a7d39918f2df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:28,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742141_1317 (size=13187) 2024-11-16T05:57:28,366 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/ef87a42643664c2f9b222ec80401e2b6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ef87a42643664c2f9b222ec80401e2b6 2024-11-16T05:57:28,375 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into ef87a42643664c2f9b222ec80401e2b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:28,375 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:28,375 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736648271; duration=0sec 2024-11-16T05:57:28,375 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:28,375 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:28,375 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:28,375 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/aff4b48a09d64b7295b4590a82945684 is 50, key is test_row_0/C:col10/1731736647405/Put/seqid=0 2024-11-16T05:57:28,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736647502, max timestamp=1731736648376Num uniques 1018; 2024-11-16T05:57:28,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T05:57:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736647502, max timestamp=1731736648376Num uniques 1018; 2024-11-16T05:57:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736647502, max timestamp=1731736648376Num uniques 1018; 2024-11-16T05:57:28,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T05:57:28,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T05:57:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742142_1318 (size=13187) 2024-11-16T05:57:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:28,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:28,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/bc76aec37ecb484b9cac5e638d4253e2 is 50, key is test_row_0/A:col10/1731736648402/Put/seqid=0 2024-11-16T05:57:28,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742143_1319 (size=14741) 2024-11-16T05:57:28,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736648465, max timestamp=1731736648478Num uniques 859; 2024-11-16T05:57:28,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736648465, max timestamp=1731736648478Num uniques 859; 2024-11-16T05:57:28,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:28,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:28,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736648465, max timestamp=1731736648478Num uniques 859; 2024-11-16T05:57:28,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736708517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736708520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736708522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T05:57:28,603 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:28,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T05:57:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:28,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:28,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:28,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736708633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736708633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736708633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736708652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736708663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:28,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:57:28,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,803 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/aff4b48a09d64b7295b4590a82945684 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/aff4b48a09d64b7295b4590a82945684 2024-11-16T05:57:28,809 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into aff4b48a09d64b7295b4590a82945684(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:28,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:28,809 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736648271; duration=0sec 2024-11-16T05:57:28,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:28,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:28,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/bc76aec37ecb484b9cac5e638d4253e2 2024-11-16T05:57:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736708843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736708843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736708844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:28,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6e122659f08e4ee5b9a9d9b42f77a0de is 50, key is test_row_0/B:col10/1731736648402/Put/seqid=0 2024-11-16T05:57:28,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742144_1320 (size=12301) 2024-11-16T05:57:28,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:57:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:28,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:29,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:57:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736709153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736709153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736709153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:57:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:29,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:29,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6e122659f08e4ee5b9a9d9b42f77a0de 2024-11-16T05:57:29,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/8daef1d701cd4ca5bd81632e1d4630a8 is 50, key is test_row_0/C:col10/1731736648402/Put/seqid=0 2024-11-16T05:57:29,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742145_1321 (size=12301) 2024-11-16T05:57:29,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/8daef1d701cd4ca5bd81632e1d4630a8 2024-11-16T05:57:29,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/bc76aec37ecb484b9cac5e638d4253e2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2 2024-11-16T05:57:29,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2, entries=200, sequenceid=383, filesize=14.4 K 2024-11-16T05:57:29,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/6e122659f08e4ee5b9a9d9b42f77a0de as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de 2024-11-16T05:57:29,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de, entries=150, sequenceid=383, filesize=12.0 K 2024-11-16T05:57:29,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/8daef1d701cd4ca5bd81632e1d4630a8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8 2024-11-16T05:57:29,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8, entries=150, sequenceid=383, filesize=12.0 K 2024-11-16T05:57:29,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=127.47 KB/130530 for ba572f15df9a669de020b29c7e6acd49 in 886ms, sequenceid=383, compaction requested=false 2024-11-16T05:57:29,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:29,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,382 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/0dd1d8ce8aa6442e854b28ed30e7c58e is 50, key is test_row_0/A:col10/1731736648517/Put/seqid=0 2024-11-16T05:57:29,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742146_1322 (size=12301) 2024-11-16T05:57:29,391 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/0dd1d8ce8aa6442e854b28ed30e7c58e 2024-11-16T05:57:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/571667529f954d37af84519392339740 is 50, key is test_row_0/B:col10/1731736648517/Put/seqid=0 2024-11-16T05:57:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742147_1323 (size=12301) 2024-11-16T05:57:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736709676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736709677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736709678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736709680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736709683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,687 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736709793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736709794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736709794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736709794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:29,805 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/571667529f954d37af84519392339740 2024-11-16T05:57:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cab897c1a0504e23bf9b589ac1a54dd8 is 50, key is test_row_0/C:col10/1731736648517/Put/seqid=0 2024-11-16T05:57:29,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742148_1324 (size=12301) 2024-11-16T05:57:29,835 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cab897c1a0504e23bf9b589ac1a54dd8 2024-11-16T05:57:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/0dd1d8ce8aa6442e854b28ed30e7c58e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e 2024-11-16T05:57:29,851 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e, entries=150, sequenceid=420, filesize=12.0 K 2024-11-16T05:57:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/571667529f954d37af84519392339740 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740 2024-11-16T05:57:29,856 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740, entries=150, sequenceid=420, filesize=12.0 K 2024-11-16T05:57:29,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cab897c1a0504e23bf9b589ac1a54dd8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8 2024-11-16T05:57:29,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8, entries=150, sequenceid=420, filesize=12.0 K 2024-11-16T05:57:29,875 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ba572f15df9a669de020b29c7e6acd49 in 493ms, sequenceid=420, compaction requested=true 2024-11-16T05:57:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T05:57:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T05:57:29,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T05:57:29,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2660 sec 2024-11-16T05:57:29,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2750 sec 2024-11-16T05:57:30,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:30,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9be42e1e26f741f8ad0c070e26a55230 is 50, key is test_row_0/A:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742149_1325 (size=12301) 2024-11-16T05:57:30,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9be42e1e26f741f8ad0c070e26a55230 2024-11-16T05:57:30,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0f2eeaa2ab254473a65113b59f289689 is 50, key is test_row_0/B:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742150_1326 (size=12301) 2024-11-16T05:57:30,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0f2eeaa2ab254473a65113b59f289689 2024-11-16T05:57:30,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/1378aca8a3444b60bdc4641f0ef92c51 is 50, key is test_row_0/C:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742151_1327 (size=12301) 2024-11-16T05:57:30,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/1378aca8a3444b60bdc4641f0ef92c51 2024-11-16T05:57:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/9be42e1e26f741f8ad0c070e26a55230 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230 2024-11-16T05:57:30,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230, entries=150, sequenceid=432, filesize=12.0 K 2024-11-16T05:57:30,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/0f2eeaa2ab254473a65113b59f289689 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689 2024-11-16T05:57:30,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689, entries=150, sequenceid=432, filesize=12.0 K 2024-11-16T05:57:30,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/1378aca8a3444b60bdc4641f0ef92c51 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51 2024-11-16T05:57:30,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51, entries=150, sequenceid=432, filesize=12.0 K 2024-11-16T05:57:30,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ba572f15df9a669de020b29c7e6acd49 in 143ms, sequenceid=432, compaction requested=true 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:30,149 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:30,149 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:30,152 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:30,152 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:30,152 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:30,152 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:30,152 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:30,152 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:30,152 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ef87a42643664c2f9b222ec80401e2b6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=48.9 K 2024-11-16T05:57:30,152 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/be73e2ff283d4b2b8fea74f5ad2ba9b8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=51.3 K 2024-11-16T05:57:30,153 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ef87a42643664c2f9b222ec80401e2b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:30,154 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting be73e2ff283d4b2b8fea74f5ad2ba9b8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:30,154 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e122659f08e4ee5b9a9d9b42f77a0de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731736648389 2024-11-16T05:57:30,154 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 571667529f954d37af84519392339740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731736648507 2024-11-16T05:57:30,154 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc76aec37ecb484b9cac5e638d4253e2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731736648376 2024-11-16T05:57:30,155 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f2eeaa2ab254473a65113b59f289689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:30,155 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd1d8ce8aa6442e854b28ed30e7c58e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731736648507 2024-11-16T05:57:30,155 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9be42e1e26f741f8ad0c070e26a55230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:30,176 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:30,176 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/32ce475f79244238b8f9752bba9cead1 is 50, key is test_row_0/B:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,181 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:30,182 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/4e9902bbd01147599ba89f2e159ffed0 is 50, key is test_row_0/A:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:30,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T05:57:30,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736650029, max timestamp=1731736650206Num uniques 859; 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:30,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:30,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742153_1329 (size=13323) 2024-11-16T05:57:30,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742152_1328 (size=13323) 2024-11-16T05:57:30,238 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/32ce475f79244238b8f9752bba9cead1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/32ce475f79244238b8f9752bba9cead1 2024-11-16T05:57:30,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/1e0ecad6004c4c4496354ff9638b0263 is 50, key is test_row_0/A:col10/1731736650206/Put/seqid=0 2024-11-16T05:57:30,249 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 32ce475f79244238b8f9752bba9cead1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:30,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:30,249 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=12, startTime=1731736650149; duration=0sec 2024-11-16T05:57:30,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:30,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:30,249 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:30,250 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:30,250 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:30,251 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:30,251 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/aff4b48a09d64b7295b4590a82945684, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=48.9 K 2024-11-16T05:57:30,251 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting aff4b48a09d64b7295b4590a82945684, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731736647405 2024-11-16T05:57:30,252 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daef1d701cd4ca5bd81632e1d4630a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731736648389 2024-11-16T05:57:30,253 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting cab897c1a0504e23bf9b589ac1a54dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731736648507 2024-11-16T05:57:30,254 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1378aca8a3444b60bdc4641f0ef92c51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,284 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#288 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:30,285 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/5cd932f719674cd6a62bafdfd4fc5563 is 50, key is test_row_0/C:col10/1731736649678/Put/seqid=0 2024-11-16T05:57:30,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742154_1330 (size=12301) 2024-11-16T05:57:30,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/1e0ecad6004c4c4496354ff9638b0263 2024-11-16T05:57:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742155_1331 (size=13323) 2024-11-16T05:57:30,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b2e227508f7240cba360f06a1a6cf294 is 50, key is test_row_0/B:col10/1731736650206/Put/seqid=0 2024-11-16T05:57:30,349 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/5cd932f719674cd6a62bafdfd4fc5563 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5cd932f719674cd6a62bafdfd4fc5563 2024-11-16T05:57:30,358 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into 5cd932f719674cd6a62bafdfd4fc5563(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:30,358 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:30,358 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=12, startTime=1731736650149; duration=0sec 2024-11-16T05:57:30,359 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:30,359 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742156_1332 (size=12301) 2024-11-16T05:57:30,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b2e227508f7240cba360f06a1a6cf294 2024-11-16T05:57:30,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cc0a505266474162a44c15d1ef124ee1 is 50, key is test_row_0/C:col10/1731736650206/Put/seqid=0 2024-11-16T05:57:30,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742157_1333 (size=12301) 2024-11-16T05:57:30,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cc0a505266474162a44c15d1ef124ee1 2024-11-16T05:57:30,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/1e0ecad6004c4c4496354ff9638b0263 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263 2024-11-16T05:57:30,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263, entries=150, sequenceid=453, filesize=12.0 K 2024-11-16T05:57:30,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b2e227508f7240cba360f06a1a6cf294 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294 2024-11-16T05:57:30,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294, entries=150, sequenceid=453, filesize=12.0 K 2024-11-16T05:57:30,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/cc0a505266474162a44c15d1ef124ee1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1 2024-11-16T05:57:30,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1, entries=150, sequenceid=453, filesize=12.0 K 2024-11-16T05:57:30,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=93.93 KB/96180 for ba572f15df9a669de020b29c7e6acd49 in 259ms, sequenceid=453, compaction requested=false 2024-11-16T05:57:30,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:30,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:57:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:30,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5bf9ee3345934ed29e79bc1ff366055d is 50, key is test_row_0/A:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:30,628 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/4e9902bbd01147599ba89f2e159ffed0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/4e9902bbd01147599ba89f2e159ffed0 2024-11-16T05:57:30,634 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 4e9902bbd01147599ba89f2e159ffed0(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:30,634 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:30,634 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=12, startTime=1731736650149; duration=0sec 2024-11-16T05:57:30,634 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:30,634 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:30,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742158_1334 (size=12301) 2024-11-16T05:57:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T05:57:30,758 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:30,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T05:57:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:57:30,761 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:30,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:30,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:57:30,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T05:57:30,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:30,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:30,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:30,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:30,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:30,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736710923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736710924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736710924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:30,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736710933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5bf9ee3345934ed29e79bc1ff366055d 2024-11-16T05:57:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:57:31,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T05:57:31,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:31,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:31,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:31,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d621a76599b342f79c6c77a015529669 is 50, key is test_row_0/B:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:31,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742159_1335 (size=12301) 2024-11-16T05:57:31,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d621a76599b342f79c6c77a015529669 2024-11-16T05:57:31,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e0e6ec3de1234a29bdd032b6436891bf is 50, key is test_row_0/C:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742160_1336 (size=12301) 2024-11-16T05:57:31,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e0e6ec3de1234a29bdd032b6436891bf 2024-11-16T05:57:31,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5bf9ee3345934ed29e79bc1ff366055d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d 2024-11-16T05:57:31,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d, entries=150, sequenceid=474, filesize=12.0 K 2024-11-16T05:57:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/d621a76599b342f79c6c77a015529669 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669 2024-11-16T05:57:31,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669, entries=150, sequenceid=474, filesize=12.0 K 2024-11-16T05:57:31,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/e0e6ec3de1234a29bdd032b6436891bf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf 2024-11-16T05:57:31,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf, entries=150, sequenceid=474, filesize=12.0 K 2024-11-16T05:57:31,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ba572f15df9a669de020b29c7e6acd49 in 636ms, sequenceid=474, compaction requested=true 2024-11-16T05:57:31,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:31,213 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:31,214 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:31,215 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:31,215 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,215 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/4e9902bbd01147599ba89f2e159ffed0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.0 K 2024-11-16T05:57:31,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:31,215 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9902bbd01147599ba89f2e159ffed0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:31,216 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e0ecad6004c4c4496354ff9638b0263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731736650088 2024-11-16T05:57:31,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:31,216 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:31,216 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf9ee3345934ed29e79bc1ff366055d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:31,218 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:31,218 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:31,218 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,218 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/32ce475f79244238b8f9752bba9cead1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.0 K 2024-11-16T05:57:31,219 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 32ce475f79244238b8f9752bba9cead1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:31,220 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e227508f7240cba360f06a1a6cf294, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731736650088 2024-11-16T05:57:31,221 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d621a76599b342f79c6c77a015529669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:31,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,230 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:31,239 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:31,240 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5b3a153f7d0d44759a2bb1803db053f8 is 50, key is test_row_0/A:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:31,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/974adc7ce0894a8e8bc449c3b968900d is 50, key is test_row_0/A:col10/1731736650601/Put/seqid=0 2024-11-16T05:57:31,256 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:31,256 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/1b8aaf2898174552832886948dd89624 is 50, key is test_row_0/B:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:31,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742162_1338 (size=12301) 2024-11-16T05:57:31,287 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/974adc7ce0894a8e8bc449c3b968900d 2024-11-16T05:57:31,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736711285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736711286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736711286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736711290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742161_1337 (size=13425) 2024-11-16T05:57:31,320 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/5b3a153f7d0d44759a2bb1803db053f8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5b3a153f7d0d44759a2bb1803db053f8 2024-11-16T05:57:31,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into 5b3a153f7d0d44759a2bb1803db053f8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:31,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:31,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736651213; duration=0sec 2024-11-16T05:57:31,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:31,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:31,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:31,327 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:31,327 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:31,327 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,327 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5cd932f719674cd6a62bafdfd4fc5563, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.0 K 2024-11-16T05:57:31,328 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cd932f719674cd6a62bafdfd4fc5563, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731736649674 2024-11-16T05:57:31,328 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc0a505266474162a44c15d1ef124ee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731736650088 2024-11-16T05:57:31,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e6ec3de1234a29bdd032b6436891bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742163_1339 (size=13425) 2024-11-16T05:57:31,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/1b8aaf2898174552832886948dd89624 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/1b8aaf2898174552832886948dd89624 2024-11-16T05:57:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34e97fc9583440218b96e9b86aee925a is 50, key is test_row_0/B:col10/1731736650601/Put/seqid=0 2024-11-16T05:57:31,351 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 1b8aaf2898174552832886948dd89624(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:31,351 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:31,351 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736651216; duration=0sec 2024-11-16T05:57:31,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:31,352 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:31,363 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:31,364 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f0b47e73110945b0b3404b5d2d045124 is 50, key is test_row_0/C:col10/1731736650575/Put/seqid=0 2024-11-16T05:57:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:57:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736711403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736711403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736711403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736711403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742164_1340 (size=12301) 2024-11-16T05:57:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742165_1341 (size=13425) 2024-11-16T05:57:31,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736711614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736711614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736711614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736711614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44188 deadline: 1731736711693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,695 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in ba572f15df9a669de020b29c7e6acd49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,815 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34e97fc9583440218b96e9b86aee925a 2024-11-16T05:57:31,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/ebbb533280ee4931953d8670fa2d4c74 is 50, key is test_row_0/C:col10/1731736650601/Put/seqid=0 2024-11-16T05:57:31,840 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/f0b47e73110945b0b3404b5d2d045124 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f0b47e73110945b0b3404b5d2d045124 2024-11-16T05:57:31,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742166_1342 (size=12301) 2024-11-16T05:57:31,842 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/ebbb533280ee4931953d8670fa2d4c74 2024-11-16T05:57:31,850 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into f0b47e73110945b0b3404b5d2d045124(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:31,850 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:31,850 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736651235; duration=0sec 2024-11-16T05:57:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/974adc7ce0894a8e8bc449c3b968900d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d 2024-11-16T05:57:31,850 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:31,851 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:31,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d, entries=150, sequenceid=488, filesize=12.0 K 2024-11-16T05:57:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/34e97fc9583440218b96e9b86aee925a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a 2024-11-16T05:57:31,860 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a, entries=150, sequenceid=488, filesize=12.0 K 2024-11-16T05:57:31,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/ebbb533280ee4931953d8670fa2d4c74 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74 2024-11-16T05:57:31,866 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74, entries=150, sequenceid=488, filesize=12.0 K 2024-11-16T05:57:31,867 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ba572f15df9a669de020b29c7e6acd49 in 637ms, sequenceid=488, compaction requested=false 2024-11-16T05:57:31,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:31,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:31,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T05:57:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T05:57:31,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T05:57:31,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-16T05:57:31,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1180 sec 2024-11-16T05:57:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T05:57:31,893 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:31,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T05:57:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:31,898 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:31,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:31,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:31,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:31,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/731c04ecf7204de5aa36e801d49b551f is 50, key is test_row_0/A:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:31,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742167_1343 (size=14741) 2024-11-16T05:57:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736711933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736711934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736711935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736711938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736712043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736712043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736712043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736712053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:32,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736712253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736712253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736712253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736712263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/731c04ecf7204de5aa36e801d49b551f 2024-11-16T05:57:32,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/35289c74ff4840f9891855ec9967896a is 50, key is test_row_0/B:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742168_1344 (size=12301) 2024-11-16T05:57:32,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:32,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736712563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736712563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736712563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736712573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:32,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/35289c74ff4840f9891855ec9967896a 2024-11-16T05:57:32,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/6e1cec1b0dfe442eb04d076b16d3edc5 is 50, key is test_row_0/C:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:32,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742169_1345 (size=12301) 2024-11-16T05:57:32,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:32,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:32,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:32,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:33,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44152 deadline: 1731736713072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731736713073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44200 deadline: 1731736713073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44242 deadline: 1731736713083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:57:33,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:33,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. as already flushing 2024-11-16T05:57:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:33,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/6e1cec1b0dfe442eb04d076b16d3edc5 2024-11-16T05:57:33,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/731c04ecf7204de5aa36e801d49b551f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f 2024-11-16T05:57:33,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f, entries=200, sequenceid=511, filesize=14.4 K 2024-11-16T05:57:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/35289c74ff4840f9891855ec9967896a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a 2024-11-16T05:57:33,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T05:57:33,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/6e1cec1b0dfe442eb04d076b16d3edc5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5 2024-11-16T05:57:33,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T05:57:33,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ba572f15df9a669de020b29c7e6acd49 in 1290ms, sequenceid=511, compaction requested=true 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:33,214 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:33,214 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ba572f15df9a669de020b29c7e6acd49:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:33,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/B is initiating minor compaction (all files) 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/A is initiating minor compaction (all files) 2024-11-16T05:57:33,216 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/B in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,216 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/A in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,216 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/1b8aaf2898174552832886948dd89624, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.1 K 2024-11-16T05:57:33,216 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5b3a153f7d0d44759a2bb1803db053f8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=39.5 K 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8aaf2898174552832886948dd89624, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:33,216 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b3a153f7d0d44759a2bb1803db053f8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:33,218 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974adc7ce0894a8e8bc449c3b968900d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731736650599 2024-11-16T05:57:33,218 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e97fc9583440218b96e9b86aee925a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731736650599 2024-11-16T05:57:33,218 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731c04ecf7204de5aa36e801d49b551f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731736651282 2024-11-16T05:57:33,218 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 35289c74ff4840f9891855ec9967896a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731736651282 2024-11-16T05:57:33,225 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:33,226 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/c0dc4b56c3e54e88ac889315b1ba35f1 is 50, key is test_row_0/A:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:33,227 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:33,227 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/992ac5cb1b264ff9b70e16d0085de7ae is 50, key is test_row_0/B:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:33,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742170_1346 (size=13527) 2024-11-16T05:57:33,259 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/992ac5cb1b264ff9b70e16d0085de7ae as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/992ac5cb1b264ff9b70e16d0085de7ae 2024-11-16T05:57:33,270 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/B of ba572f15df9a669de020b29c7e6acd49 into 992ac5cb1b264ff9b70e16d0085de7ae(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:33,270 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:33,270 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/B, priority=13, startTime=1731736653214; duration=0sec 2024-11-16T05:57:33,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:33,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:B 2024-11-16T05:57:33,271 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:33,273 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:33,273 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): ba572f15df9a669de020b29c7e6acd49/C is initiating minor compaction (all files) 2024-11-16T05:57:33,273 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ba572f15df9a669de020b29c7e6acd49/C in TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T05:57:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:57:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:33,274 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f0b47e73110945b0b3404b5d2d045124, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp, totalSize=37.1 K 2024-11-16T05:57:33,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742171_1347 (size=13527) 2024-11-16T05:57:33,275 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f0b47e73110945b0b3404b5d2d045124, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731736650251 2024-11-16T05:57:33,277 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ebbb533280ee4931953d8670fa2d4c74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731736650599 2024-11-16T05:57:33,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e1cec1b0dfe442eb04d076b16d3edc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731736651282 2024-11-16T05:57:33,286 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ba572f15df9a669de020b29c7e6acd49#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/c5f26270ccf74db58a425e1322486012 is 50, key is test_row_0/A:col10/1731736651935/Put/seqid=0 2024-11-16T05:57:33,287 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/19c8d7e868c948ba8c167b3dd31e7e04 is 50, key is test_row_0/C:col10/1731736651287/Put/seqid=0 2024-11-16T05:57:33,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742172_1348 (size=12301) 2024-11-16T05:57:33,311 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/c5f26270ccf74db58a425e1322486012 2024-11-16T05:57:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742173_1349 (size=13527) 2024-11-16T05:57:33,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b036544beb0e4aeb919cabb47ad4525c is 50, key is test_row_0/B:col10/1731736651935/Put/seqid=0 2024-11-16T05:57:33,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742174_1350 (size=12301) 2024-11-16T05:57:33,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b036544beb0e4aeb919cabb47ad4525c 2024-11-16T05:57:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/73eafff4bde64f3db59c380efe8841ab is 50, key is test_row_0/C:col10/1731736651935/Put/seqid=0 2024-11-16T05:57:33,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742175_1351 (size=12301) 2024-11-16T05:57:33,377 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/73eafff4bde64f3db59c380efe8841ab 2024-11-16T05:57:33,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/c5f26270ccf74db58a425e1322486012 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c5f26270ccf74db58a425e1322486012 2024-11-16T05:57:33,386 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c5f26270ccf74db58a425e1322486012, entries=150, sequenceid=524, filesize=12.0 K 2024-11-16T05:57:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/b036544beb0e4aeb919cabb47ad4525c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b036544beb0e4aeb919cabb47ad4525c 2024-11-16T05:57:33,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b036544beb0e4aeb919cabb47ad4525c, entries=150, sequenceid=524, filesize=12.0 K 2024-11-16T05:57:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/73eafff4bde64f3db59c380efe8841ab as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73eafff4bde64f3db59c380efe8841ab 2024-11-16T05:57:33,397 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73eafff4bde64f3db59c380efe8841ab, entries=150, sequenceid=524, filesize=12.0 K 2024-11-16T05:57:33,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ba572f15df9a669de020b29c7e6acd49 in 124ms, sequenceid=524, compaction requested=false 2024-11-16T05:57:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T05:57:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T05:57:33,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T05:57:33,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-11-16T05:57:33,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5060 sec 2024-11-16T05:57:33,554 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-16T05:57:33,554 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:33,554 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,554 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:33,555 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-16T05:57:33,555 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:33,555 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:33,555 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,556 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-16T05:57:33,556 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:33,556 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,556 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:33,557 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-16T05:57:33,557 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-16T05:57:33,557 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:33,557 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:33,557 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,557 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,557 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,557 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:33,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:33,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:33,678 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/c0dc4b56c3e54e88ac889315b1ba35f1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c0dc4b56c3e54e88ac889315b1ba35f1 2024-11-16T05:57:33,682 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/A of ba572f15df9a669de020b29c7e6acd49 into c0dc4b56c3e54e88ac889315b1ba35f1(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:33,682 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:33,682 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/A, priority=13, startTime=1731736653214; duration=0sec 2024-11-16T05:57:33,682 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:33,682 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:A 2024-11-16T05:57:33,738 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/19c8d7e868c948ba8c167b3dd31e7e04 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/19c8d7e868c948ba8c167b3dd31e7e04 2024-11-16T05:57:33,743 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba572f15df9a669de020b29c7e6acd49/C of ba572f15df9a669de020b29c7e6acd49 into 19c8d7e868c948ba8c167b3dd31e7e04(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:33,744 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ba572f15df9a669de020b29c7e6acd49: 2024-11-16T05:57:33,744 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., storeName=ba572f15df9a669de020b29c7e6acd49/C, priority=13, startTime=1731736653214; duration=0sec 2024-11-16T05:57:33,744 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:33,744 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ba572f15df9a669de020b29c7e6acd49:C 2024-11-16T05:57:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T05:57:34,043 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:34,084 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-16T05:57:34,084 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-16T05:57:34,084 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:34,084 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:34,084 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,084 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,084 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,084 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:34,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:34,094 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-16T05:57:34,094 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-16T05:57:34,094 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:34,094 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:34,094 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,094 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,094 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,094 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:34,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:34,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:35,736 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-16T05:57:35,736 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:57:35,736 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:35,737 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:35,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2704 2024-11-16T05:57:35,737 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2706 2024-11-16T05:57:35,738 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2668 2024-11-16T05:57:35,738 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2740 2024-11-16T05:57:35,738 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2685 2024-11-16T05:57:35,738 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:57:35,738 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:57:35,738 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:57:35,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:35,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:35,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:57:35,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:35,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736655745"}]},"ts":"1731736655745"} 2024-11-16T05:57:35,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:57:35,747 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:57:35,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:57:35,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, UNASSIGN}] 2024-11-16T05:57:35,750 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, UNASSIGN 2024-11-16T05:57:35,751 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=ba572f15df9a669de020b29c7e6acd49, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:35,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, UNASSIGN because future has completed 2024-11-16T05:57:35,753 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:57:35,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:35,906 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing ba572f15df9a669de020b29c7e6acd49, disabling compactions & flushes 2024-11-16T05:57:35,906 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. after waiting 0 ms 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:35,906 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing ba572f15df9a669de020b29c7e6acd49 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=A 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:35,906 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=B 2024-11-16T05:57:35,907 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:35,907 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ba572f15df9a669de020b29c7e6acd49, store=C 2024-11-16T05:57:35,907 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:35,910 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/269417a562994f58accd0a272c94af04 is 50, key is test_row_0/A:col10/1731736654093/Put/seqid=0 2024-11-16T05:57:35,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742176_1352 (size=12301) 2024-11-16T05:57:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:36,314 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/269417a562994f58accd0a272c94af04 2024-11-16T05:57:36,320 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/ec1190a9fb824c128bd5ad869ae95ba8 is 50, key is test_row_0/B:col10/1731736654093/Put/seqid=0 2024-11-16T05:57:36,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742177_1353 (size=12301) 2024-11-16T05:57:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:36,725 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/ec1190a9fb824c128bd5ad869ae95ba8 2024-11-16T05:57:36,732 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/574761977c3c464fbda341532af9131a is 50, key is test_row_0/C:col10/1731736654093/Put/seqid=0 2024-11-16T05:57:36,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742178_1354 (size=12301) 2024-11-16T05:57:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:37,140 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/574761977c3c464fbda341532af9131a 2024-11-16T05:57:37,145 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/A/269417a562994f58accd0a272c94af04 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/269417a562994f58accd0a272c94af04 2024-11-16T05:57:37,149 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/269417a562994f58accd0a272c94af04, entries=150, sequenceid=534, filesize=12.0 K 2024-11-16T05:57:37,150 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/B/ec1190a9fb824c128bd5ad869ae95ba8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ec1190a9fb824c128bd5ad869ae95ba8 2024-11-16T05:57:37,158 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ec1190a9fb824c128bd5ad869ae95ba8, entries=150, sequenceid=534, filesize=12.0 K 2024-11-16T05:57:37,160 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/.tmp/C/574761977c3c464fbda341532af9131a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/574761977c3c464fbda341532af9131a 2024-11-16T05:57:37,165 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/574761977c3c464fbda341532af9131a, entries=150, sequenceid=534, filesize=12.0 K 2024-11-16T05:57:37,166 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ba572f15df9a669de020b29c7e6acd49 in 1260ms, sequenceid=534, compaction requested=true 2024-11-16T05:57:37,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6a5af520a526410f8f3b36970a693339, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/85a7c6f695804bdca5734a722061b744, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/8fa070aaf6544b0da299f78facce28e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/32712daf6a514e8a9449050bb5dffe9f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/438bfb04fbc94c009dad3db9d6884993, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/fdaf8f9f1f69485ba6a54d4d0252a66b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/33e16b42d0da422fb9f0e3eb1d36734c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/be73e2ff283d4b2b8fea74f5ad2ba9b8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/4e9902bbd01147599ba89f2e159ffed0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5b3a153f7d0d44759a2bb1803db053f8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f] to archive 2024-11-16T05:57:37,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:37,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9f7f8c0964c742f480338eab903a3610 2024-11-16T05:57:37,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6309fe72fa064648a88b297f324bd92b 2024-11-16T05:57:37,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b1266c17085f4dd9a80fc5fab46fdbcf 2024-11-16T05:57:37,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6a5af520a526410f8f3b36970a693339 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6a5af520a526410f8f3b36970a693339 2024-11-16T05:57:37,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5740684e6c754a7183429acc8a8970ea 2024-11-16T05:57:37,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/241f8bf18071452b96c59ecee537f04d 2024-11-16T05:57:37,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/85a7c6f695804bdca5734a722061b744 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/85a7c6f695804bdca5734a722061b744 2024-11-16T05:57:37,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/93217eb5a83841edafc886463486e440 2024-11-16T05:57:37,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e3ed45f4f99a417a966beeb022f64fd5 2024-11-16T05:57:37,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/8fa070aaf6544b0da299f78facce28e9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/8fa070aaf6544b0da299f78facce28e9 2024-11-16T05:57:37,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/867dd79d375542d4b67e974435d6ecde 2024-11-16T05:57:37,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/46f8ffc1a31b42eab65e9e7297ae1401 2024-11-16T05:57:37,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/32712daf6a514e8a9449050bb5dffe9f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/32712daf6a514e8a9449050bb5dffe9f 2024-11-16T05:57:37,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/f0108ba0bd6240ada82a2e28cc932031 2024-11-16T05:57:37,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/ada198a8f75e444aab6c226c9421918b 2024-11-16T05:57:37,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/3f9eb034eecc4a1fa2defc8050f8e30c 2024-11-16T05:57:37,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/438bfb04fbc94c009dad3db9d6884993 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/438bfb04fbc94c009dad3db9d6884993 2024-11-16T05:57:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/20d73e873ccd4fd89e786238a0c9d323 2024-11-16T05:57:37,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/b2d9cc2b674241f285493ad0354ca4d2 2024-11-16T05:57:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/edb68e08d4a94c368d313ba4bdc0661a 2024-11-16T05:57:37,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/fdaf8f9f1f69485ba6a54d4d0252a66b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/fdaf8f9f1f69485ba6a54d4d0252a66b 2024-11-16T05:57:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/e8f9a16406b64aebadfdb97151cdf47e 2024-11-16T05:57:37,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/33e16b42d0da422fb9f0e3eb1d36734c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/33e16b42d0da422fb9f0e3eb1d36734c 2024-11-16T05:57:37,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/a425a7c9288a438395dffb3604cded6d 2024-11-16T05:57:37,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/65431e8dd06745709298c20b39d85101 2024-11-16T05:57:37,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/be73e2ff283d4b2b8fea74f5ad2ba9b8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/be73e2ff283d4b2b8fea74f5ad2ba9b8 2024-11-16T05:57:37,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/6ac918d4538f4aa6926a4d3431bd59ab 2024-11-16T05:57:37,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/bc76aec37ecb484b9cac5e638d4253e2 2024-11-16T05:57:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/0dd1d8ce8aa6442e854b28ed30e7c58e 2024-11-16T05:57:37,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/4e9902bbd01147599ba89f2e159ffed0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/4e9902bbd01147599ba89f2e159ffed0 2024-11-16T05:57:37,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/9be42e1e26f741f8ad0c070e26a55230 2024-11-16T05:57:37,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/1e0ecad6004c4c4496354ff9638b0263 2024-11-16T05:57:37,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5b3a153f7d0d44759a2bb1803db053f8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5b3a153f7d0d44759a2bb1803db053f8 2024-11-16T05:57:37,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/5bf9ee3345934ed29e79bc1ff366055d 2024-11-16T05:57:37,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/974adc7ce0894a8e8bc449c3b968900d 2024-11-16T05:57:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/731c04ecf7204de5aa36e801d49b551f 2024-11-16T05:57:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e7003e771494b179e462e46b016f255, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2e1d24b24dd1429a8d510a0ea7d71798, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d5921819cd7748cd9c486359b5252ab9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/9cf949827e2940a28b0afd800b587840, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6f04c2b10b0141b4832f8d2d952a77dc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d3fb7a547cb348b1aeffc8151b33a679, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d501708d78da4bbeb724e2027f0631c7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ef87a42643664c2f9b222ec80401e2b6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/32ce475f79244238b8f9752bba9cead1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/1b8aaf2898174552832886948dd89624, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a] to archive 2024-11-16T05:57:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4aaf246a61ea42c8827995aaf0649719 2024-11-16T05:57:37,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/4d4f4f77e555434881edefc77d33d040 2024-11-16T05:57:37,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e7003e771494b179e462e46b016f255 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e7003e771494b179e462e46b016f255 2024-11-16T05:57:37,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e5ca9e771f404e13b25523eecec5ce2d 2024-11-16T05:57:37,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/524cb0b49ac04420a1cb610b3d6cc1a5 2024-11-16T05:57:37,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2e1d24b24dd1429a8d510a0ea7d71798 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2e1d24b24dd1429a8d510a0ea7d71798 2024-11-16T05:57:37,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/484f1fd01b3846f8808d55232bbd2772 2024-11-16T05:57:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8a0bf9e4c84b4684bbc6f7535a6983f1 2024-11-16T05:57:37,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d5921819cd7748cd9c486359b5252ab9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d5921819cd7748cd9c486359b5252ab9 2024-11-16T05:57:37,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0d91bbdc1430438a94e05a91df2f7479 2024-11-16T05:57:37,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/23a1268bacf94ca69a61eae948241a0a 2024-11-16T05:57:37,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34eb40eadbf34456972c910191c2217f 2024-11-16T05:57:37,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/9cf949827e2940a28b0afd800b587840 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/9cf949827e2940a28b0afd800b587840 2024-11-16T05:57:37,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/43d9d1d6bfef4ccda0e3b435b9b21e2e 2024-11-16T05:57:37,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/97b144f6845640fe80cee2eaee09e36c 2024-11-16T05:57:37,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6f04c2b10b0141b4832f8d2d952a77dc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6f04c2b10b0141b4832f8d2d952a77dc 2024-11-16T05:57:37,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/8f106ac3858d46fe85afc144582ad633 2024-11-16T05:57:37,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/2280d763b4e648f68d972d4aa433a2cc 2024-11-16T05:57:37,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/e60e3692063d4b22b567c04c9e98862f 2024-11-16T05:57:37,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d3fb7a547cb348b1aeffc8151b33a679 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d3fb7a547cb348b1aeffc8151b33a679 2024-11-16T05:57:37,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/607849ecd78e4eb9b93e3a22030145dd 2024-11-16T05:57:37,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/afc1a658f1c145109c95e4ac0094b1db 2024-11-16T05:57:37,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d501708d78da4bbeb724e2027f0631c7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d501708d78da4bbeb724e2027f0631c7 2024-11-16T05:57:37,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/446ed88f8c7c42a7921ccc9bda7de455 2024-11-16T05:57:37,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/cf54c5ffd37d4a77846aa404a2364aeb 2024-11-16T05:57:37,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ef87a42643664c2f9b222ec80401e2b6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ef87a42643664c2f9b222ec80401e2b6 2024-11-16T05:57:37,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/a0fd34573feb46dbba00e201323faa4e 2024-11-16T05:57:37,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/6e122659f08e4ee5b9a9d9b42f77a0de 2024-11-16T05:57:37,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/571667529f954d37af84519392339740 2024-11-16T05:57:37,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/32ce475f79244238b8f9752bba9cead1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/32ce475f79244238b8f9752bba9cead1 2024-11-16T05:57:37,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/0f2eeaa2ab254473a65113b59f289689 2024-11-16T05:57:37,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b2e227508f7240cba360f06a1a6cf294 2024-11-16T05:57:37,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/1b8aaf2898174552832886948dd89624 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/1b8aaf2898174552832886948dd89624 2024-11-16T05:57:37,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/d621a76599b342f79c6c77a015529669 2024-11-16T05:57:37,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/34e97fc9583440218b96e9b86aee925a 2024-11-16T05:57:37,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/35289c74ff4840f9891855ec9967896a 2024-11-16T05:57:37,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e5bd0d800ad54097a56e5e3a2b588d0b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/de14d6083d9a4a59ab34d25ae9652926, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1139ff0597b14d43a17abed7e280d2df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f25eec9db15840bfb7a1b8eda451aaad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f68e01b4a0554d9aba18516c5da31c57, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73bcf19796f049dbaaff60b615bf13ef, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/7fdee599df6b447e9f9e352446562b81, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/aff4b48a09d64b7295b4590a82945684, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5cd932f719674cd6a62bafdfd4fc5563, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f0b47e73110945b0b3404b5d2d045124, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5] to archive 2024-11-16T05:57:37,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:57:37,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/99187437a64b4e949472e7a7def47e4c 2024-11-16T05:57:37,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a62155599eb04390925bf8dccf96d6b4 2024-11-16T05:57:37,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e5bd0d800ad54097a56e5e3a2b588d0b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e5bd0d800ad54097a56e5e3a2b588d0b 2024-11-16T05:57:37,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5c1087f24eb94402b0d46ed69a9253b2 2024-11-16T05:57:37,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c21141eedad34743a6ee32fe631793d0 2024-11-16T05:57:37,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/de14d6083d9a4a59ab34d25ae9652926 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/de14d6083d9a4a59ab34d25ae9652926 2024-11-16T05:57:37,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a35c36fbf7e44d9caf205fdb36c03786 2024-11-16T05:57:37,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/616d6b1ad3194a06b7d8358d578461aa 2024-11-16T05:57:37,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1139ff0597b14d43a17abed7e280d2df to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1139ff0597b14d43a17abed7e280d2df 2024-11-16T05:57:37,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/b84e059262934c029cb557b0b9fb41cb 2024-11-16T05:57:37,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/3349d3ebe0594e67bbaac2f3166c5fbd 2024-11-16T05:57:37,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/44c47f8efcba4c3cb4b0b36fe57b4fe3 2024-11-16T05:57:37,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f25eec9db15840bfb7a1b8eda451aaad to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f25eec9db15840bfb7a1b8eda451aaad 2024-11-16T05:57:37,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c694a4b9709743bab27620b3e0e559d8 2024-11-16T05:57:37,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/d7495ca36bcd412ea2de19b124e4c793 2024-11-16T05:57:37,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f68e01b4a0554d9aba18516c5da31c57 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f68e01b4a0554d9aba18516c5da31c57 2024-11-16T05:57:37,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/223295b18a4c4c12942edf598372a9b1 2024-11-16T05:57:37,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/c242d7f685c7484c81aeb4a3b1542402 2024-11-16T05:57:37,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/433d50efbcc248f0a39f722f72e05609 2024-11-16T05:57:37,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73bcf19796f049dbaaff60b615bf13ef to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73bcf19796f049dbaaff60b615bf13ef 2024-11-16T05:57:37,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/38c9d8e82ec9457987a81ee6e473df0b 2024-11-16T05:57:37,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/2dccc441c7184f38909fd23fc16c241e 2024-11-16T05:57:37,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/7fdee599df6b447e9f9e352446562b81 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/7fdee599df6b447e9f9e352446562b81 2024-11-16T05:57:37,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/a7e29928a3b44034bf3865f3dfb452fd 2024-11-16T05:57:37,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e4328c910d0640d58148f81757b16202 2024-11-16T05:57:37,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/aff4b48a09d64b7295b4590a82945684 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/aff4b48a09d64b7295b4590a82945684 2024-11-16T05:57:37,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e89f36c888c04bc8bac2a7d39918f2df 2024-11-16T05:57:37,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/8daef1d701cd4ca5bd81632e1d4630a8 2024-11-16T05:57:37,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cab897c1a0504e23bf9b589ac1a54dd8 2024-11-16T05:57:37,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5cd932f719674cd6a62bafdfd4fc5563 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/5cd932f719674cd6a62bafdfd4fc5563 2024-11-16T05:57:37,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/1378aca8a3444b60bdc4641f0ef92c51 2024-11-16T05:57:37,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/cc0a505266474162a44c15d1ef124ee1 2024-11-16T05:57:37,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f0b47e73110945b0b3404b5d2d045124 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/f0b47e73110945b0b3404b5d2d045124 2024-11-16T05:57:37,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/e0e6ec3de1234a29bdd032b6436891bf 2024-11-16T05:57:37,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/ebbb533280ee4931953d8670fa2d4c74 2024-11-16T05:57:37,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/6e1cec1b0dfe442eb04d076b16d3edc5 2024-11-16T05:57:37,407 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/recovered.edits/537.seqid, newMaxSeqId=537, maxSeqId=1 2024-11-16T05:57:37,408 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. 2024-11-16T05:57:37,408 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for ba572f15df9a669de020b29c7e6acd49: Waiting for close lock at 1731736655906Running coprocessor pre-close hooks at 1731736655906Disabling compacts and flushes for region at 1731736655906Disabling writes for close at 1731736655906Obtaining lock to block concurrent updates at 1731736655906Preparing flush snapshotting stores in ba572f15df9a669de020b29c7e6acd49 at 1731736655906Finished memstore snapshotting TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731736655907 (+1 ms)Flushing stores of TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49. at 1731736655907Flushing ba572f15df9a669de020b29c7e6acd49/A: creating writer at 1731736655907Flushing ba572f15df9a669de020b29c7e6acd49/A: appending metadata at 1731736655910 (+3 ms)Flushing ba572f15df9a669de020b29c7e6acd49/A: closing flushed file at 1731736655910Flushing ba572f15df9a669de020b29c7e6acd49/B: creating writer at 1731736656318 (+408 ms)Flushing ba572f15df9a669de020b29c7e6acd49/B: appending metadata at 1731736656320 (+2 ms)Flushing ba572f15df9a669de020b29c7e6acd49/B: closing flushed file at 1731736656320Flushing ba572f15df9a669de020b29c7e6acd49/C: creating writer at 1731736656729 (+409 ms)Flushing ba572f15df9a669de020b29c7e6acd49/C: appending metadata at 1731736656732 (+3 ms)Flushing ba572f15df9a669de020b29c7e6acd49/C: closing flushed file at 1731736656732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dee2604: reopening flushed file at 1731736657145 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f5ccf73: reopening flushed file at 1731736657150 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fad4afb: reopening flushed file at 1731736657158 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ba572f15df9a669de020b29c7e6acd49 in 1260ms, sequenceid=534, compaction requested=true at 1731736657166 (+8 ms)Writing region close event to WAL at 1731736657404 (+238 ms)Running coprocessor post-close hooks at 1731736657408 (+4 ms)Closed at 1731736657408 2024-11-16T05:57:37,410 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:37,410 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=ba572f15df9a669de020b29c7e6acd49, regionState=CLOSED 2024-11-16T05:57:37,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:37,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T05:57:37,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure ba572f15df9a669de020b29c7e6acd49, server=3456ee6a3164,45735,1731736558702 in 1.6600 sec 2024-11-16T05:57:37,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-16T05:57:37,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ba572f15df9a669de020b29c7e6acd49, UNASSIGN in 1.6660 sec 2024-11-16T05:57:37,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T05:57:37,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6690 sec 2024-11-16T05:57:37,420 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736657420"}]},"ts":"1731736657420"} 2024-11-16T05:57:37,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:57:37,422 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:57:37,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6820 sec 2024-11-16T05:57:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T05:57:37,894 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:37,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:57:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,896 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:57:37,897 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,900 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:37,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/recovered.edits] 2024-11-16T05:57:37,905 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/269417a562994f58accd0a272c94af04 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/269417a562994f58accd0a272c94af04 2024-11-16T05:57:37,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c0dc4b56c3e54e88ac889315b1ba35f1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c0dc4b56c3e54e88ac889315b1ba35f1 2024-11-16T05:57:37,907 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c5f26270ccf74db58a425e1322486012 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/A/c5f26270ccf74db58a425e1322486012 2024-11-16T05:57:37,909 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/992ac5cb1b264ff9b70e16d0085de7ae to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/992ac5cb1b264ff9b70e16d0085de7ae 2024-11-16T05:57:37,914 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b036544beb0e4aeb919cabb47ad4525c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/b036544beb0e4aeb919cabb47ad4525c 2024-11-16T05:57:37,915 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ec1190a9fb824c128bd5ad869ae95ba8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/B/ec1190a9fb824c128bd5ad869ae95ba8 2024-11-16T05:57:37,918 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/19c8d7e868c948ba8c167b3dd31e7e04 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/19c8d7e868c948ba8c167b3dd31e7e04 2024-11-16T05:57:37,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/574761977c3c464fbda341532af9131a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/574761977c3c464fbda341532af9131a 2024-11-16T05:57:37,920 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73eafff4bde64f3db59c380efe8841ab to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/C/73eafff4bde64f3db59c380efe8841ab 2024-11-16T05:57:37,936 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/recovered.edits/537.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49/recovered.edits/537.seqid 2024-11-16T05:57:37,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/ba572f15df9a669de020b29c7e6acd49 2024-11-16T05:57:37,937 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:57:37,939 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,943 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:57:37,945 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:57:37,946 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,946 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:57:37,946 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736657946"}]},"ts":"9223372036854775807"} 2024-11-16T05:57:37,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:57:37,949 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ba572f15df9a669de020b29c7e6acd49, NAME => 'TestAcidGuarantees,,1731736632371.ba572f15df9a669de020b29c7e6acd49.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:57:37,949 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:57:37,949 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736657949"}]},"ts":"9223372036854775807"} 2024-11-16T05:57:37,952 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:57:37,953 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:37,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-16T05:57:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T05:57:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:57:38,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:38,012 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1574838696_22 at /127.0.0.1:58124 [Waiting for operation #168] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-62023640_22 at /127.0.0.1:39828 [Waiting for operation #578] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2714 (was 2955) 2024-11-16T05:57:38,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=2714 2024-11-16T05:57:38,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:57:38,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:57:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:38,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:57:38,029 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:38,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-16T05:57:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-16T05:57:38,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:57:38,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742179_1355 (size=1061) 2024-11-16T05:57:38,042 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:57:38,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742180_1356 (size=53) 2024-11-16T05:57:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-16T05:57:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-16T05:57:38,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:57:38,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4b8bf1d7473bdc01de0055e89dea26d3, disabling compactions & flushes 2024-11-16T05:57:38,449 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. after waiting 0 ms 2024-11-16T05:57:38,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,449 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4b8bf1d7473bdc01de0055e89dea26d3: Waiting for close lock at 1731736658448Disabling compacts and flushes for region at 1731736658448Disabling writes for close at 1731736658449 (+1 ms)Writing region close event to WAL at 1731736658449Closed at 1731736658449 2024-11-16T05:57:38,450 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:57:38,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736658450"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736658450"}]},"ts":"1731736658450"} 2024-11-16T05:57:38,451 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:57:38,452 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:57:38,452 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736658452"}]},"ts":"1731736658452"} 2024-11-16T05:57:38,454 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:57:38,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, ASSIGN}] 2024-11-16T05:57:38,455 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, ASSIGN 2024-11-16T05:57:38,456 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:57:38,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:57:38,607 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:38,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, ASSIGN because future has completed 2024-11-16T05:57:38,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-16T05:57:38,763 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,764 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:57:38,764 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,764 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:57:38,764 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,764 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,766 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,767 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:38,767 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName A 2024-11-16T05:57:38,767 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:38,767 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:38,767 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,768 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:38,768 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName B 2024-11-16T05:57:38,768 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:38,769 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:38,769 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,770 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:38,770 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName C 2024-11-16T05:57:38,770 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:38,770 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:38,770 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,770 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,771 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,771 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,772 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,772 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,772 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:57:38,773 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,775 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:57:38,775 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 4b8bf1d7473bdc01de0055e89dea26d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63500250, jitterRate=-0.053772538900375366}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:57:38,775 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:38,776 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 4b8bf1d7473bdc01de0055e89dea26d3: Running coprocessor pre-open hook at 1731736658764Writing region info on filesystem at 1731736658764Initializing all the Stores at 1731736658765 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736658765Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736658765Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736658765Cleaning up temporary data from old regions at 1731736658772 (+7 ms)Running coprocessor post-open hooks at 1731736658776 (+4 ms)Region opened successfully at 1731736658776 2024-11-16T05:57:38,777 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., pid=83, masterSystemTime=1731736658760 2024-11-16T05:57:38,779 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,779 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:38,779 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:38,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:38,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T05:57:38,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 in 172 msec 2024-11-16T05:57:38,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-16T05:57:38,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, ASSIGN in 328 msec 2024-11-16T05:57:38,785 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:57:38,785 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736658785"}]},"ts":"1731736658785"} 2024-11-16T05:57:38,787 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:57:38,787 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:57:38,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 761 msec 2024-11-16T05:57:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-16T05:57:39,163 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:39,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:39,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:39,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:39,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a76c47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:39,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11430e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:39,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:39,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:39,172 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:57:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:57:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3590e8ad 2024-11-16T05:57:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:57:39,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:57:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:57:39,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:57:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:57:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742181_1357 (size=1097) 2024-11-16T05:57:39,588 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T05:57:39,588 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T05:57:39,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:57:39,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE}] 2024-11-16T05:57:39,594 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE 2024-11-16T05:57:39,594 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:39,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE because future has completed 2024-11-16T05:57:39,596 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:57:39,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:39,749 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:39,749 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:57:39,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 4b8bf1d7473bdc01de0055e89dea26d3, disabling compactions & flushes 2024-11-16T05:57:39,750 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:39,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:39,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. after waiting 0 ms 2024-11-16T05:57:39,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:39,754 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:57:39,754 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:39,754 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 4b8bf1d7473bdc01de0055e89dea26d3: Waiting for close lock at 1731736659749Running coprocessor pre-close hooks at 1731736659749Disabling compacts and flushes for region at 1731736659749Disabling writes for close at 1731736659750 (+1 ms)Writing region close event to WAL at 1731736659750Running coprocessor post-close hooks at 1731736659754 (+4 ms)Closed at 1731736659754 2024-11-16T05:57:39,754 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 4b8bf1d7473bdc01de0055e89dea26d3 to self. 2024-11-16T05:57:39,756 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:39,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=CLOSED 2024-11-16T05:57:39,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:39,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T05:57:39,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 in 164 msec 2024-11-16T05:57:39,762 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=true 2024-11-16T05:57:39,913 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:39,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE because future has completed 2024-11-16T05:57:39,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:57:40,070 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,070 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:57:40,070 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,070 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:57:40,070 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,071 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,072 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,073 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:40,073 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName A 2024-11-16T05:57:40,074 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:40,075 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:40,075 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,076 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:40,076 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName B 2024-11-16T05:57:40,076 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:40,076 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:40,077 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,077 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:57:40,077 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8bf1d7473bdc01de0055e89dea26d3 columnFamilyName C 2024-11-16T05:57:40,077 DEBUG [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:40,078 INFO [StoreOpener-4b8bf1d7473bdc01de0055e89dea26d3-1 {}] regionserver.HStore(327): Store=4b8bf1d7473bdc01de0055e89dea26d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:57:40,078 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,079 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:57:40,082 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,082 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 4b8bf1d7473bdc01de0055e89dea26d3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69638614, jitterRate=0.03769621253013611}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:57:40,082 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,083 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 4b8bf1d7473bdc01de0055e89dea26d3: Running coprocessor pre-open hook at 1731736660071Writing region info on filesystem at 1731736660071Initializing all the Stores at 1731736660071Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736660072 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736660072Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736660072Cleaning up temporary data from old regions at 1731736660080 (+8 ms)Running coprocessor post-open hooks at 1731736660082 (+2 ms)Region opened successfully at 1731736660083 (+1 ms) 2024-11-16T05:57:40,084 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., pid=88, masterSystemTime=1731736660066 2024-11-16T05:57:40,086 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,086 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:57:40,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-16T05:57:40,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 in 173 msec 2024-11-16T05:57:40,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-16T05:57:40,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, REOPEN/MOVE in 497 msec 2024-11-16T05:57:40,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T05:57:40,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-16T05:57:40,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-11-16T05:57:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T05:57:40,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:40,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,099 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d1bdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,101 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227bda8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,103 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364409d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260a2267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,107 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21973d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597be5df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,111 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258b6f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4e8f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,115 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4daca595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,117 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f7f195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,125 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,126 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1ad77b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b7d297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3cb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4f80a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,134 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a8fbc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311cc88f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c421a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,139 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d84aef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:57:40,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:57:40,141 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32088b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:57:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:57:40,143 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:57:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c897c4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:57:40,144 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,144 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,165 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,177 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,177 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,181 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,181 DEBUG [Thread-1631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:40,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,198 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,198 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,200 DEBUG [Thread-1616 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,200 DEBUG [Thread-1616 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,213 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,213 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:57:40,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:57:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:57:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-16T05:57:40,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:57:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:40,218 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:57:40,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:40,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:40,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116590dad7a516442779f4438bf5f6669a5_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736660181/Put/seqid=0 2024-11-16T05:57:40,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736660243, max timestamp=1731736660269Num uniques 859; 2024-11-16T05:57:40,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:40,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736660243, max timestamp=1731736660269Num uniques 859; 2024-11-16T05:57:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:40,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736660243, max timestamp=1731736660273Num uniques 859; 2024-11-16T05:57:40,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:57:40,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742182_1358 (size=12154) 2024-11-16T05:57:40,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:40,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116590dad7a516442779f4438bf5f6669a5_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116590dad7a516442779f4438bf5f6669a5_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a7b729f2288342809fcbe1a8189a7764, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:40,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a7b729f2288342809fcbe1a8189a7764 is 175, key is test_row_0/A:col10/1731736660181/Put/seqid=0 2024-11-16T05:57:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:40,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736720319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736720319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736720320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742183_1359 (size=30955) 2024-11-16T05:57:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736720322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736720320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736720443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736720453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736720454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736720454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736720454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736720663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736720665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736720673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736720673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736720674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:40,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a7b729f2288342809fcbe1a8189a7764 2024-11-16T05:57:40,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/74eefa9a9986415d8a9cfd1d5b171add is 50, key is test_row_0/B:col10/1731736660181/Put/seqid=0 2024-11-16T05:57:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742184_1360 (size=12001) 2024-11-16T05:57:40,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/74eefa9a9986415d8a9cfd1d5b171add 2024-11-16T05:57:40,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:40,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/d86cae383f514239a796fd347acf192a is 50, key is test_row_0/C:col10/1731736660181/Put/seqid=0 2024-11-16T05:57:40,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742185_1361 (size=12001) 2024-11-16T05:57:40,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/d86cae383f514239a796fd347acf192a 2024-11-16T05:57:40,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a7b729f2288342809fcbe1a8189a7764 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764 2024-11-16T05:57:40,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764, entries=150, sequenceid=14, filesize=30.2 K 2024-11-16T05:57:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/74eefa9a9986415d8a9cfd1d5b171add as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add 2024-11-16T05:57:40,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:57:40,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/d86cae383f514239a796fd347acf192a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a 2024-11-16T05:57:40,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:57:40,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4b8bf1d7473bdc01de0055e89dea26d3 in 746ms, sequenceid=14, compaction requested=false 2024-11-16T05:57:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T05:57:40,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:40,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:40,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:40,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736720999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736721001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736721003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111614ce6be019c540d2bdb5f7f53f2dd9e1_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736660318/Put/seqid=0 2024-11-16T05:57:41,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736721007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736721004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742186_1362 (size=14594) 2024-11-16T05:57:41,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:41,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111614ce6be019c540d2bdb5f7f53f2dd9e1_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111614ce6be019c540d2bdb5f7f53f2dd9e1_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:41,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/76be090ef7aa4671bdc08f997125382e, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:41,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/76be090ef7aa4671bdc08f997125382e is 175, key is test_row_0/A:col10/1731736660318/Put/seqid=0 2024-11-16T05:57:41,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742187_1363 (size=39549) 2024-11-16T05:57:41,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/76be090ef7aa4671bdc08f997125382e 2024-11-16T05:57:41,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/89981394e9994503812f62468cb55d5c is 50, key is test_row_0/B:col10/1731736660318/Put/seqid=0 2024-11-16T05:57:41,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742188_1364 (size=12001) 2024-11-16T05:57:41,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/89981394e9994503812f62468cb55d5c 2024-11-16T05:57:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736721113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736721113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736721113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/e3db32787a004aa88c164294905e85e7 is 50, key is test_row_0/C:col10/1731736660318/Put/seqid=0 2024-11-16T05:57:41,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742189_1365 (size=12001) 2024-11-16T05:57:41,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736721132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:41,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:41,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736721333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736721333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736721333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:41,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736721353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736721353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:41,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/e3db32787a004aa88c164294905e85e7 2024-11-16T05:57:41,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/76be090ef7aa4671bdc08f997125382e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e 2024-11-16T05:57:41,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e, entries=200, sequenceid=50, filesize=38.6 K 2024-11-16T05:57:41,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/89981394e9994503812f62468cb55d5c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c 2024-11-16T05:57:41,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T05:57:41,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/e3db32787a004aa88c164294905e85e7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7 2024-11-16T05:57:41,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T05:57:41,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 4b8bf1d7473bdc01de0055e89dea26d3 in 577ms, sequenceid=50, compaction requested=false 2024-11-16T05:57:41,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:41,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-16T05:57:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:41,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116036d0bc951164951a7a27809001b6b60_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736661002/Put/seqid=0 2024-11-16T05:57:41,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742190_1366 (size=12154) 2024-11-16T05:57:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:41,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736661729, max timestamp=1731736661733Num uniques 859; 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736661729, max timestamp=1731736661733Num uniques 859; 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736661729, max timestamp=1731736661733Num uniques 859; 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:41,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736721782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736721782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736721802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736721803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736721804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736721912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736721912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736721933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736721933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736721933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:42,024 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116036d0bc951164951a7a27809001b6b60_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116036d0bc951164951a7a27809001b6b60_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/236e67039f4a4b02ba93f1fab97110b1, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/236e67039f4a4b02ba93f1fab97110b1 is 175, key is test_row_0/A:col10/1731736661002/Put/seqid=0 2024-11-16T05:57:42,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742191_1367 (size=30955) 2024-11-16T05:57:42,032 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/236e67039f4a4b02ba93f1fab97110b1 2024-11-16T05:57:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7e78b601f4cf4f5997b4e112b53268c0 is 50, key is test_row_0/B:col10/1731736661002/Put/seqid=0 2024-11-16T05:57:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742192_1368 (size=12001) 2024-11-16T05:57:42,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7e78b601f4cf4f5997b4e112b53268c0 2024-11-16T05:57:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1864ac1fd00f41589337e939f96d8c87 is 50, key is test_row_0/C:col10/1731736661002/Put/seqid=0 2024-11-16T05:57:42,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742193_1369 (size=12001) 2024-11-16T05:57:42,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1864ac1fd00f41589337e939f96d8c87 2024-11-16T05:57:42,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/236e67039f4a4b02ba93f1fab97110b1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1 2024-11-16T05:57:42,066 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1, entries=150, sequenceid=59, filesize=30.2 K 2024-11-16T05:57:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7e78b601f4cf4f5997b4e112b53268c0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0 2024-11-16T05:57:42,073 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T05:57:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1864ac1fd00f41589337e939f96d8c87 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87 2024-11-16T05:57:42,078 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T05:57:42,079 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 4b8bf1d7473bdc01de0055e89dea26d3 in 479ms, sequenceid=59, compaction requested=true 2024-11-16T05:57:42,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:42,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-16T05:57:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-16T05:57:42,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-16T05:57:42,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-16T05:57:42,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.8700 sec 2024-11-16T05:57:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:42,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736661780, max timestamp=1731736662142Num uniques 1068; 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736661780, max timestamp=1731736662142Num uniques 1068; 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736661780, max timestamp=1731736662142Num uniques 1068; 2024-11-16T05:57:42,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167adc49efa7d9429ca3d765eb1977dcc9_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:42,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742194_1370 (size=17034) 2024-11-16T05:57:42,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:42,169 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167adc49efa7d9429ca3d765eb1977dcc9_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adc49efa7d9429ca3d765eb1977dcc9_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:42,170 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6ffc0eb60ca8499592b796e1d8ba9278, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6ffc0eb60ca8499592b796e1d8ba9278 is 175, key is test_row_0/A:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742195_1371 (size=48139) 2024-11-16T05:57:42,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6ffc0eb60ca8499592b796e1d8ba9278 2024-11-16T05:57:42,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 is 50, key is test_row_0/B:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:42,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742196_1372 (size=12001) 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736662175, max timestamp=1731736662202Num uniques 859; 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736662175, max timestamp=1731736662202Num uniques 859; 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731736662175, max timestamp=1731736662202Num uniques 859; 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:57:42,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T05:57:42,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736722247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736722247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736722248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736722259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736722259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T05:57:42,373 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:42,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-16T05:57:42,375 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:42,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:42,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736722362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736722373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736722373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736722373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736722382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:42,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T05:57:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:42,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 2024-11-16T05:57:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736722582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736722593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736722593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736722593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/f05f7c68b47342ad905e4100e68bf352 is 50, key is test_row_0/C:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742197_1373 (size=12001) 2024-11-16T05:57:42,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/f05f7c68b47342ad905e4100e68bf352 2024-11-16T05:57:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6ffc0eb60ca8499592b796e1d8ba9278 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278 2024-11-16T05:57:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736722613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278, entries=250, sequenceid=98, filesize=47.0 K 2024-11-16T05:57:42,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 2024-11-16T05:57:42,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:57:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/f05f7c68b47342ad905e4100e68bf352 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352 2024-11-16T05:57:42,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:57:42,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for 4b8bf1d7473bdc01de0055e89dea26d3 in 513ms, sequenceid=98, compaction requested=true 2024-11-16T05:57:42,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:42,658 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:42,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:42,659 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:42,659 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:42,660 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:42,660 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,660 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=146.1 K 2024-11-16T05:57:42,660 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,660 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278] 2024-11-16T05:57:42,660 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:42,660 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:42,660 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,660 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=46.9 K 2024-11-16T05:57:42,660 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b729f2288342809fcbe1a8189a7764, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736660181 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76be090ef7aa4671bdc08f997125382e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736660318 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 74eefa9a9986415d8a9cfd1d5b171add, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736660181 2024-11-16T05:57:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 236e67039f4a4b02ba93f1fab97110b1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736660999 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 89981394e9994503812f62468cb55d5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736660318 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e78b601f4cf4f5997b4e112b53268c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736660999 2024-11-16T05:57:42,661 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ffc0eb60ca8499592b796e1d8ba9278, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661780 2024-11-16T05:57:42,662 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa7fe31dc1a4cad92bb4c66dd5a00d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661801 2024-11-16T05:57:42,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T05:57:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:42,687 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-16T05:57:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:42,689 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:42,690 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/104cfa235eae41d0a7d6317e7b7172c4 is 50, key is test_row_0/B:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:42,698 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,720 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411162ae39542f75c489ea38628a546161877_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,722 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411162ae39542f75c489ea38628a546161877_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,722 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162ae39542f75c489ea38628a546161877_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:42,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742198_1374 (size=12139) 2024-11-16T05:57:42,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163855ce48f463436093c0c0ae7270a450_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736662223/Put/seqid=0 2024-11-16T05:57:42,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742200_1376 (size=12254) 2024-11-16T05:57:42,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742199_1375 (size=4469) 2024-11-16T05:57:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:42,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:42,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736722926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736722927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736722933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736722929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736722943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:42,977 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:57:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:43,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736723043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736723063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736723063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,134 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/104cfa235eae41d0a7d6317e7b7172c4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/104cfa235eae41d0a7d6317e7b7172c4 2024-11-16T05:57:43,138 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into 104cfa235eae41d0a7d6317e7b7172c4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:43,138 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:43,139 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=12, startTime=1731736662659; duration=0sec 2024-11-16T05:57:43,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:43,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:43,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:43,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:43,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:43,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:43,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=46.9 K 2024-11-16T05:57:43,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d86cae383f514239a796fd347acf192a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736660181 2024-11-16T05:57:43,142 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e3db32787a004aa88c164294905e85e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736660318 2024-11-16T05:57:43,142 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1864ac1fd00f41589337e939f96d8c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731736660999 2024-11-16T05:57:43,142 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f05f7c68b47342ad905e4100e68bf352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661801 2024-11-16T05:57:43,149 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:43,149 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c2546821bd9f400e94ba485d6cc1d76e is 50, key is test_row_0/C:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:43,158 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#325 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:43,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742201_1377 (size=12139) 2024-11-16T05:57:43,158 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a230683e376f4f468a48af9ab88a7a13 is 175, key is test_row_0/A:col10/1731736661801/Put/seqid=0 2024-11-16T05:57:43,168 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163855ce48f463436093c0c0ae7270a450_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163855ce48f463436093c0c0ae7270a450_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/f89ce9ab119a454995f91613c0b18a38, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/f89ce9ab119a454995f91613c0b18a38 is 175, key is test_row_0/A:col10/1731736662223/Put/seqid=0 2024-11-16T05:57:43,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742202_1378 (size=31093) 2024-11-16T05:57:43,177 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a230683e376f4f468a48af9ab88a7a13 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13 2024-11-16T05:57:43,182 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into a230683e376f4f468a48af9ab88a7a13(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:43,182 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:43,182 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=12, startTime=1731736662658; duration=0sec 2024-11-16T05:57:43,182 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:43,182 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:43,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742203_1379 (size=31055) 2024-11-16T05:57:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736723263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736723283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736723283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736723443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736723463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:43,562 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c2546821bd9f400e94ba485d6cc1d76e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c2546821bd9f400e94ba485d6cc1d76e 2024-11-16T05:57:43,566 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into c2546821bd9f400e94ba485d6cc1d76e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:43,566 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:43,566 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=12, startTime=1731736662659; duration=0sec 2024-11-16T05:57:43,566 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:43,566 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:43,588 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/f89ce9ab119a454995f91613c0b18a38 2024-11-16T05:57:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736723583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/409a768ff31f4ea2918c25311540e8aa is 50, key is test_row_0/B:col10/1731736662223/Put/seqid=0 2024-11-16T05:57:43,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736723603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736723603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:43,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742204_1380 (size=12101) 2024-11-16T05:57:44,013 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/409a768ff31f4ea2918c25311540e8aa 2024-11-16T05:57:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b242f3ac6d134c11b16dfd9de2814212 is 50, key is test_row_0/C:col10/1731736662223/Put/seqid=0 2024-11-16T05:57:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742205_1381 (size=12101) 2024-11-16T05:57:44,027 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b242f3ac6d134c11b16dfd9de2814212 2024-11-16T05:57:44,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/f89ce9ab119a454995f91613c0b18a38 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38 2024-11-16T05:57:44,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38, entries=150, sequenceid=135, filesize=30.3 K 2024-11-16T05:57:44,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/409a768ff31f4ea2918c25311540e8aa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa 2024-11-16T05:57:44,044 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa, entries=150, sequenceid=135, filesize=11.8 K 2024-11-16T05:57:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b242f3ac6d134c11b16dfd9de2814212 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212 2024-11-16T05:57:44,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212, entries=150, sequenceid=135, filesize=11.8 K 2024-11-16T05:57:44,054 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1366ms, sequenceid=135, compaction requested=false 2024-11-16T05:57:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-16T05:57:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-16T05:57:44,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-16T05:57:44,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6790 sec 2024-11-16T05:57:44,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.6840 sec 2024-11-16T05:57:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:44,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:44,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d42eb6f51bf8433a9883ee6ab2e21bcb_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742206_1382 (size=14794) 2024-11-16T05:57:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736664149, max timestamp=1731736664162Num uniques 859; 2024-11-16T05:57:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736664149, max timestamp=1731736664162Num uniques 859; 2024-11-16T05:57:44,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736664149, max timestamp=1731736664162Num uniques 859; 2024-11-16T05:57:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736724213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736724333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736724343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736724343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736724463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,474 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736724473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,484 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:44,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:44,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d42eb6f51bf8433a9883ee6ab2e21bcb_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d42eb6f51bf8433a9883ee6ab2e21bcb_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:44,531 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/2cf8a3fa29334499a5c920a94cabf6f3, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:44,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/2cf8a3fa29334499a5c920a94cabf6f3 is 175, key is test_row_0/A:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T05:57:44,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:44,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-16T05:57:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:44,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:44,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:44,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:44,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742207_1383 (size=39749) 2024-11-16T05:57:44,540 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/2cf8a3fa29334499a5c920a94cabf6f3 2024-11-16T05:57:44,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/9cbf120ddbd7425b9509617a323532ec is 50, key is test_row_0/B:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:44,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736724543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742208_1384 (size=12151) 2024-11-16T05:57:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736724554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736724555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:44,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T05:57:44,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T05:57:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736724863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736724872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736724883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:44,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/9cbf120ddbd7425b9509617a323532ec 2024-11-16T05:57:44,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1d17eceed8874eec9818ed61db95f843 is 50, key is test_row_0/C:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742209_1385 (size=12151) 2024-11-16T05:57:44,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1d17eceed8874eec9818ed61db95f843 2024-11-16T05:57:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/2cf8a3fa29334499a5c920a94cabf6f3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3 2024-11-16T05:57:44,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3, entries=200, sequenceid=148, filesize=38.8 K 2024-11-16T05:57:44,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/9cbf120ddbd7425b9509617a323532ec as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec 2024-11-16T05:57:44,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T05:57:44,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/1d17eceed8874eec9818ed61db95f843 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843 2024-11-16T05:57:44,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T05:57:44,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T05:57:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4b8bf1d7473bdc01de0055e89dea26d3 in 886ms, sequenceid=148, compaction requested=true 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:44,995 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:44,995 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:44,997 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:44,998 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:44,998 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,998 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=99.5 K 2024-11-16T05:57:44,998 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:44,998 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3] 2024-11-16T05:57:45,001 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:45,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a230683e376f4f468a48af9ab88a7a13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661801 2024-11-16T05:57:45,001 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:45,001 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:45,002 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/104cfa235eae41d0a7d6317e7b7172c4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=35.5 K 2024-11-16T05:57:45,002 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89ce9ab119a454995f91613c0b18a38, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731736662223 2024-11-16T05:57:45,002 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf8a3fa29334499a5c920a94cabf6f3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:45,002 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 104cfa235eae41d0a7d6317e7b7172c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661801 2024-11-16T05:57:45,002 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 409a768ff31f4ea2918c25311540e8aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731736662223 2024-11-16T05:57:45,003 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cbf120ddbd7425b9509617a323532ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:45,009 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:45,009 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/ec3f5a4bd3724437b43db409c48b2d11 is 50, key is test_row_0/B:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:45,010 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:45,038 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166a3ba058fae24f5284f54454ffabb442_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:45,040 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166a3ba058fae24f5284f54454ffabb442_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:45,040 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166a3ba058fae24f5284f54454ffabb442_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:45,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742210_1386 (size=12391) 2024-11-16T05:57:45,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742211_1387 (size=4469) 2024-11-16T05:57:45,051 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#334 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:45,051 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c20cc09797e94e7e882ff6fb0e064045 is 175, key is test_row_0/A:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:45,056 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/ec3f5a4bd3724437b43db409c48b2d11 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ec3f5a4bd3724437b43db409c48b2d11 2024-11-16T05:57:45,067 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into ec3f5a4bd3724437b43db409c48b2d11(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:45,067 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:45,068 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=13, startTime=1731736664995; duration=0sec 2024-11-16T05:57:45,068 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:45,068 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:45,068 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:45,069 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:45,069 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:45,070 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:45,070 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c2546821bd9f400e94ba485d6cc1d76e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=35.5 K 2024-11-16T05:57:45,070 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c2546821bd9f400e94ba485d6cc1d76e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736661801 2024-11-16T05:57:45,070 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b242f3ac6d134c11b16dfd9de2814212, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731736662223 2024-11-16T05:57:45,071 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d17eceed8874eec9818ed61db95f843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:45,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742212_1388 (size=31345) 2024-11-16T05:57:45,087 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c20cc09797e94e7e882ff6fb0e064045 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045 2024-11-16T05:57:45,087 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:45,088 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/01005995bb47404d9226268acd40811b is 50, key is test_row_0/C:col10/1731736664105/Put/seqid=0 2024-11-16T05:57:45,097 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into c20cc09797e94e7e882ff6fb0e064045(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:45,097 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:45,097 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=13, startTime=1731736664995; duration=0sec 2024-11-16T05:57:45,097 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:45,097 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742213_1389 (size=12391) 2024-11-16T05:57:45,109 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/01005995bb47404d9226268acd40811b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/01005995bb47404d9226268acd40811b 2024-11-16T05:57:45,115 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into 01005995bb47404d9226268acd40811b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:45,115 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:45,115 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=13, startTime=1731736664995; duration=0sec 2024-11-16T05:57:45,115 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:45,115 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:45,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T05:57:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:45,151 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:45,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5fd597fea704340ac50a6dc613f0120_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736664227/Put/seqid=0 2024-11-16T05:57:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:45,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742214_1390 (size=12304) 2024-11-16T05:57:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:45,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736725404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736725410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736725417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736725523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736725523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736725543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:45,583 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5fd597fea704340ac50a6dc613f0120_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5fd597fea704340ac50a6dc613f0120_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c63dbb2007984e16855cf982a8226e97, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c63dbb2007984e16855cf982a8226e97 is 175, key is test_row_0/A:col10/1731736664227/Put/seqid=0 2024-11-16T05:57:45,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742215_1391 (size=31105) 2024-11-16T05:57:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736725743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736725744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736725762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:45,992 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c63dbb2007984e16855cf982a8226e97 2024-11-16T05:57:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/6a6659d742ba4bb7bc1980088f78847d is 50, key is test_row_0/B:col10/1731736664227/Put/seqid=0 2024-11-16T05:57:46,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742216_1392 (size=12151) 2024-11-16T05:57:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736726063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736726063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736726083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,407 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/6a6659d742ba4bb7bc1980088f78847d 2024-11-16T05:57:46,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5d258ff907db492183ab37af339d3110 is 50, key is test_row_0/C:col10/1731736664227/Put/seqid=0 2024-11-16T05:57:46,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742217_1393 (size=12151) 2024-11-16T05:57:46,428 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5d258ff907db492183ab37af339d3110 2024-11-16T05:57:46,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c63dbb2007984e16855cf982a8226e97 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97 2024-11-16T05:57:46,437 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97, entries=150, sequenceid=187, filesize=30.4 K 2024-11-16T05:57:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/6a6659d742ba4bb7bc1980088f78847d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d 2024-11-16T05:57:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T05:57:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5d258ff907db492183ab37af339d3110 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110 2024-11-16T05:57:46,446 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110, entries=150, sequenceid=187, filesize=11.9 K 2024-11-16T05:57:46,447 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1296ms, sequenceid=187, compaction requested=false 2024-11-16T05:57:46,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-16T05:57:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-16T05:57:46,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T05:57:46,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-16T05:57:46,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.9170 sec 2024-11-16T05:57:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:46,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:46,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111611a2a2c26af84a70bd7195f1c0208128_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:46,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742218_1394 (size=14794) 2024-11-16T05:57:46,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:46,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111611a2a2c26af84a70bd7195f1c0208128_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a2a2c26af84a70bd7195f1c0208128_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:46,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6edaf956fcf340a4b85d163ef0d328b9, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:46,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6edaf956fcf340a4b85d163ef0d328b9 is 175, key is test_row_0/A:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:46,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742219_1395 (size=39749) 2024-11-16T05:57:46,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736726583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736726583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736726587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736666561, max timestamp=1731736666590Num uniques 859; 2024-11-16T05:57:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736666561, max timestamp=1731736666590Num uniques 859; 2024-11-16T05:57:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:46,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736666561, max timestamp=1731736666590Num uniques 859; 2024-11-16T05:57:46,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T05:57:46,683 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:46,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T05:57:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:46,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:46,690 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:46,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736726696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736726702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736726708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:46,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736726823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736726823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:46,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736726923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:46,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6edaf956fcf340a4b85d163ef0d328b9 2024-11-16T05:57:46,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7f77827f690c41dda74a9bd35bde75dc is 50, key is test_row_0/B:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742220_1396 (size=12151) 2024-11-16T05:57:46,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:46,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:46,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:46,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736727033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736727042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736727246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736727343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736727362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7f77827f690c41dda74a9bd35bde75dc 2024-11-16T05:57:47,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/2f051d54c8fd42df904027ba0dfb2df0 is 50, key is test_row_0/C:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:47,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742221_1397 (size=12151) 2024-11-16T05:57:47,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736727602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,611 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736727612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,623 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736727773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/2f051d54c8fd42df904027ba0dfb2df0 2024-11-16T05:57:47,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/6edaf956fcf340a4b85d163ef0d328b9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9 2024-11-16T05:57:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:47,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9, entries=200, sequenceid=198, filesize=38.8 K 2024-11-16T05:57:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7f77827f690c41dda74a9bd35bde75dc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc 2024-11-16T05:57:47,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T05:57:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/2f051d54c8fd42df904027ba0dfb2df0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0 2024-11-16T05:57:47,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T05:57:47,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1331ms, sequenceid=198, compaction requested=true 2024-11-16T05:57:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:47,841 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:47,841 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:47,842 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,842 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,842 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ec3f5a4bd3724437b43db409c48b2d11, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=35.8 K 2024-11-16T05:57:47,842 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=99.8 K 2024-11-16T05:57:47,842 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9] 2024-11-16T05:57:47,842 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3f5a4bd3724437b43db409c48b2d11, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:47,843 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20cc09797e94e7e882ff6fb0e064045, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:47,843 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6659d742ba4bb7bc1980088f78847d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731736664208 2024-11-16T05:57:47,843 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c63dbb2007984e16855cf982a8226e97, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731736664208 2024-11-16T05:57:47,843 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f77827f690c41dda74a9bd35bde75dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:47,843 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6edaf956fcf340a4b85d163ef0d328b9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:47,848 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:47,849 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:47,850 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/dcb99aca07184cdfbac042b9a9b8c0e9 is 50, key is test_row_0/B:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:47,853 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116bf2cf9fe18d14f51bb9e25f5fd45084a_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:47,855 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116bf2cf9fe18d14f51bb9e25f5fd45084a_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:47,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742222_1398 (size=12493) 2024-11-16T05:57:47,855 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bf2cf9fe18d14f51bb9e25f5fd45084a_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:47,861 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/dcb99aca07184cdfbac042b9a9b8c0e9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/dcb99aca07184cdfbac042b9a9b8c0e9 2024-11-16T05:57:47,866 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into dcb99aca07184cdfbac042b9a9b8c0e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:47,866 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:47,866 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=13, startTime=1731736667841; duration=0sec 2024-11-16T05:57:47,867 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:47,867 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:47,867 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:47,869 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:47,870 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:47,870 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,870 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/01005995bb47404d9226268acd40811b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=35.8 K 2024-11-16T05:57:47,870 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 01005995bb47404d9226268acd40811b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731736662904 2024-11-16T05:57:47,870 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d258ff907db492183ab37af339d3110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731736664208 2024-11-16T05:57:47,871 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f051d54c8fd42df904027ba0dfb2df0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742223_1399 (size=4469) 2024-11-16T05:57:47,876 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#343 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:47,876 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c8c770cef06a4022955ef38214a10884 is 175, key is test_row_0/A:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:47,876 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:47,877 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/e74d7cddb62b4788a660910ec310149b is 50, key is test_row_0/C:col10/1731736666508/Put/seqid=0 2024-11-16T05:57:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742224_1400 (size=31447) 2024-11-16T05:57:47,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742225_1401 (size=12493) 2024-11-16T05:57:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:47,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T05:57:47,885 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c8c770cef06a4022955ef38214a10884 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:47,890 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into c8c770cef06a4022955ef38214a10884(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:47,890 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:47,890 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=13, startTime=1731736667840; duration=0sec 2024-11-16T05:57:47,890 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:47,890 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:47,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169d6c35e63089436682629cb0d3072584_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736666640/Put/seqid=0 2024-11-16T05:57:47,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742226_1402 (size=14794) 2024-11-16T05:57:47,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:47,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736727913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736727922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736728033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736728034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736728243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736728252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,286 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/e74d7cddb62b4788a660910ec310149b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e74d7cddb62b4788a660910ec310149b 2024-11-16T05:57:48,290 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into e74d7cddb62b4788a660910ec310149b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:48,290 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:48,290 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=13, startTime=1731736667841; duration=0sec 2024-11-16T05:57:48,290 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:48,290 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:48,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:48,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169d6c35e63089436682629cb0d3072584_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169d6c35e63089436682629cb0d3072584_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:48,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cafd2de0b6f7462c9506228e6042de72, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:48,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cafd2de0b6f7462c9506228e6042de72 is 175, key is test_row_0/A:col10/1731736666640/Put/seqid=0 2024-11-16T05:57:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742227_1403 (size=39749) 2024-11-16T05:57:48,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:57:48,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T05:57:48,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736728553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736728573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cafd2de0b6f7462c9506228e6042de72 2024-11-16T05:57:48,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/412941966c6046618003df0b7900d6be is 50, key is test_row_0/B:col10/1731736666640/Put/seqid=0 2024-11-16T05:57:48,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742228_1404 (size=12151) 2024-11-16T05:57:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736728802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:48,812 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:48,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:48,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736729073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736729083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:49,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/412941966c6046618003df0b7900d6be 2024-11-16T05:57:49,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 is 50, key is test_row_0/C:col10/1731736666640/Put/seqid=0 2024-11-16T05:57:49,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742229_1405 (size=12151) 2024-11-16T05:57:49,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:49,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 2024-11-16T05:57:49,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cafd2de0b6f7462c9506228e6042de72 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72 2024-11-16T05:57:49,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72, entries=200, sequenceid=236, filesize=38.8 K 2024-11-16T05:57:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/412941966c6046618003df0b7900d6be as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be 2024-11-16T05:57:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T05:57:49,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T05:57:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1666ms, sequenceid=236, compaction requested=false 2024-11-16T05:57:49,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T05:57:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:49,602 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:57:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116296e2cfdb67c421fab9e9847418b8e6b_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_1/A:col10/1731736667912/Put/seqid=0 2024-11-16T05:57:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742230_1406 (size=9814) 2024-11-16T05:57:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,667 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116296e2cfdb67c421fab9e9847418b8e6b_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296e2cfdb67c421fab9e9847418b8e6b_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/4f49057631094ee1a4173819af8ee127, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/4f49057631094ee1a4173819af8ee127 is 175, key is test_row_1/A:col10/1731736667912/Put/seqid=0 2024-11-16T05:57:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:49,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742231_1407 (size=22461) 2024-11-16T05:57:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/4f49057631094ee1a4173819af8ee127 2024-11-16T05:57:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/ca4e60098696497e9819eb3cc90c7c2a is 50, key is test_row_1/B:col10/1731736667912/Put/seqid=0 2024-11-16T05:57:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742232_1408 (size=9757) 2024-11-16T05:57:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736669733, max timestamp=1731736669772Num uniques 859; 2024-11-16T05:57:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736669733, max timestamp=1731736669772Num uniques 859; 2024-11-16T05:57:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:49,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:49,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736669733, max timestamp=1731736669772Num uniques 859; 2024-11-16T05:57:49,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736729843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:49,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736729847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736729953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736729963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736730093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,104 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736730103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,109 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,121 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/ca4e60098696497e9819eb3cc90c7c2a 2024-11-16T05:57:50,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c22f69ffe4f84fc9acb97814cc995c3f is 50, key is test_row_1/C:col10/1731736667912/Put/seqid=0 2024-11-16T05:57:50,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742233_1409 (size=9757) 2024-11-16T05:57:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736730163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736730182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736730475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736730493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,535 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c22f69ffe4f84fc9acb97814cc995c3f 2024-11-16T05:57:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/4f49057631094ee1a4173819af8ee127 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127 2024-11-16T05:57:50,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127, entries=100, sequenceid=248, filesize=21.9 K 2024-11-16T05:57:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/ca4e60098696497e9819eb3cc90c7c2a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a 2024-11-16T05:57:50,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a, entries=100, sequenceid=248, filesize=9.5 K 2024-11-16T05:57:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c22f69ffe4f84fc9acb97814cc995c3f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f 2024-11-16T05:57:50,570 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f, entries=100, sequenceid=248, filesize=9.5 K 2024-11-16T05:57:50,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4b8bf1d7473bdc01de0055e89dea26d3 in 973ms, sequenceid=248, compaction requested=true 2024-11-16T05:57:50,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:50,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:50,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T05:57:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T05:57:50,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T05:57:50,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8870 sec 2024-11-16T05:57:50,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.8960 sec 2024-11-16T05:57:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:50,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:50,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T05:57:50,843 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:50,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T05:57:50,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:50,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:50,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:50,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116168ba7738f494f0492a896692172b8c9_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:50,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742234_1410 (size=14994) 2024-11-16T05:57:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736730895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:50,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:50,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736730993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736731013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736731013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:51,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736731226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:51,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:51,262 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116168ba7738f494f0492a896692172b8c9_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116168ba7738f494f0492a896692172b8c9_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:51,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/e36260f9cb5b48dfaff89707c751afe4, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/e36260f9cb5b48dfaff89707c751afe4 is 175, key is test_row_0/A:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742235_1411 (size=39949) 2024-11-16T05:57:51,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/e36260f9cb5b48dfaff89707c751afe4 2024-11-16T05:57:51,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/f0891260f719451f94453c15791f29b0 is 50, key is test_row_0/B:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742236_1412 (size=12301) 2024-11-16T05:57:51,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:51,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736731553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:51,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/f0891260f719451f94453c15791f29b0 2024-11-16T05:57:51,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b39b2a941e59484499f02781536b1127 is 50, key is test_row_0/C:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742237_1413 (size=12301) 2024-11-16T05:57:51,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b39b2a941e59484499f02781536b1127 2024-11-16T05:57:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/e36260f9cb5b48dfaff89707c751afe4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4 2024-11-16T05:57:51,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4, entries=200, sequenceid=285, filesize=39.0 K 2024-11-16T05:57:51,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/f0891260f719451f94453c15791f29b0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0 2024-11-16T05:57:51,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:51,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-16T05:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b39b2a941e59484499f02781536b1127 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127 2024-11-16T05:57:51,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127, entries=150, sequenceid=285, filesize=12.0 K 2024-11-16T05:57:51,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4b8bf1d7473bdc01de0055e89dea26d3 in 937ms, sequenceid=285, compaction requested=true 2024-11-16T05:57:51,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:51,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:51,779 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:51,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:51,779 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:51,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:51,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:51,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:51,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:51,781 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:51,781 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:51,781 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,781 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/dcb99aca07184cdfbac042b9a9b8c0e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=45.6 K 2024-11-16T05:57:51,782 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb99aca07184cdfbac042b9a9b8c0e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:51,782 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133606 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:51,782 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:51,782 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,782 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=130.5 K 2024-11-16T05:57:51,782 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 412941966c6046618003df0b7900d6be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736666640 2024-11-16T05:57:51,782 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,782 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4] 2024-11-16T05:57:51,783 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4e60098696497e9819eb3cc90c7c2a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731736667912 2024-11-16T05:57:51,783 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c770cef06a4022955ef38214a10884, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:51,783 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f0891260f719451f94453c15791f29b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:51,783 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting cafd2de0b6f7462c9506228e6042de72, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736666640 2024-11-16T05:57:51,785 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f49057631094ee1a4173819af8ee127, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731736667912 2024-11-16T05:57:51,786 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e36260f9cb5b48dfaff89707c751afe4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:51,808 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:51,809 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/af07d626457041a7a6c5cc59620b57c3 is 50, key is test_row_0/B:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,810 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742238_1414 (size=12779) 2024-11-16T05:57:51,837 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411162000eeb0ec35427f9d077004b3dea44f_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,840 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411162000eeb0ec35427f9d077004b3dea44f_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,841 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162000eeb0ec35427f9d077004b3dea44f_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,841 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/af07d626457041a7a6c5cc59620b57c3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/af07d626457041a7a6c5cc59620b57c3 2024-11-16T05:57:51,855 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into af07d626457041a7a6c5cc59620b57c3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:51,855 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:51,855 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=12, startTime=1731736671779; duration=0sec 2024-11-16T05:57:51,855 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:51,855 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:51,855 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:51,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:51,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:51,856 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,857 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e74d7cddb62b4788a660910ec310149b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=45.6 K 2024-11-16T05:57:51,858 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e74d7cddb62b4788a660910ec310149b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731736665400 2024-11-16T05:57:51,860 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 55501db6dd7d4d6a8c3e6c6a6c1e8f20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731736666640 2024-11-16T05:57:51,860 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c22f69ffe4f84fc9acb97814cc995c3f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731736667912 2024-11-16T05:57:51,861 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b39b2a941e59484499f02781536b1127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742239_1415 (size=4469) 2024-11-16T05:57:51,865 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#355 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:51,866 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a8c29f4e679345aca000f2dacbb04bb4 is 175, key is test_row_0/A:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742240_1416 (size=31733) 2024-11-16T05:57:51,887 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:51,888 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/02f4b13784f34ef2a9692f8c568c0d72 is 50, key is test_row_0/C:col10/1731736669845/Put/seqid=0 2024-11-16T05:57:51,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742241_1417 (size=12779) 2024-11-16T05:57:51,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:51,917 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:51,924 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/02f4b13784f34ef2a9692f8c568c0d72 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/02f4b13784f34ef2a9692f8c568c0d72 2024-11-16T05:57:51,937 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into 02f4b13784f34ef2a9692f8c568c0d72(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:51,937 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:51,937 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=12, startTime=1731736671779; duration=0sec 2024-11-16T05:57:51,937 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:51,937 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c454f1e7127548d18c01c8f2151263d6_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736670868/Put/seqid=0 2024-11-16T05:57:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742242_1418 (size=12454) 2024-11-16T05:57:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:51,972 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c454f1e7127548d18c01c8f2151263d6_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c454f1e7127548d18c01c8f2151263d6_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/37824d18233a4669b62bd9bef1ce76c0, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/37824d18233a4669b62bd9bef1ce76c0 is 175, key is test_row_0/A:col10/1731736670868/Put/seqid=0 2024-11-16T05:57:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742243_1419 (size=31255) 2024-11-16T05:57:52,011 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/37824d18233a4669b62bd9bef1ce76c0 2024-11-16T05:57:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:52,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/0761f1d3d07943cf9c33bb23f53e86d0 is 50, key is test_row_0/B:col10/1731736670868/Put/seqid=0 2024-11-16T05:57:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742244_1420 (size=12301) 2024-11-16T05:57:52,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736672087, max timestamp=1731736672111Num uniques 859; 2024-11-16T05:57:52,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:52,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736672087, max timestamp=1731736672111Num uniques 859; 2024-11-16T05:57:52,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:52,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736672087, max timestamp=1731736672111Num uniques 859; 2024-11-16T05:57:52,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:57:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736732170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731736672133, max timestamp=1731736672176Num uniques 1018; 2024-11-16T05:57:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-16T05:57:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731736672133, max timestamp=1731736672176Num uniques 1018; 2024-11-16T05:57:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-16T05:57:52,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731736672133, max timestamp=1731736672176Num uniques 1018; 2024-11-16T05:57:52,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-16T05:57:52,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736732246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736732248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736732263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736732264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736732283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,289 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/a8c29f4e679345aca000f2dacbb04bb4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4 2024-11-16T05:57:52,293 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into a8c29f4e679345aca000f2dacbb04bb4(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:52,293 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:52,293 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=12, startTime=1731736671778; duration=0sec 2024-11-16T05:57:52,293 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:52,294 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736732373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736732373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736732373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736732393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,466 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/0761f1d3d07943cf9c33bb23f53e86d0 2024-11-16T05:57:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/a8adcf8e1b564d06bb79e3afa7e4292a is 50, key is test_row_0/C:col10/1731736670868/Put/seqid=0 2024-11-16T05:57:52,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736732493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742245_1421 (size=12301) 2024-11-16T05:57:52,507 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/a8adcf8e1b564d06bb79e3afa7e4292a 2024-11-16T05:57:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/37824d18233a4669b62bd9bef1ce76c0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0 2024-11-16T05:57:52,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0, entries=150, sequenceid=295, filesize=30.5 K 2024-11-16T05:57:52,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/0761f1d3d07943cf9c33bb23f53e86d0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0 2024-11-16T05:57:52,536 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T05:57:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/a8adcf8e1b564d06bb79e3afa7e4292a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a 2024-11-16T05:57:52,543 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T05:57:52,550 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 4b8bf1d7473bdc01de0055e89dea26d3 in 633ms, sequenceid=295, compaction requested=false 2024-11-16T05:57:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T05:57:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T05:57:52,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T05:57:52,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7060 sec 2024-11-16T05:57:52,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.7150 sec 2024-11-16T05:57:52,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:52,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166b4796f8d427445ea5c6bcfe3a00c323_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:52,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736732603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736732616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736732617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736732619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742246_1422 (size=14994) 2024-11-16T05:57:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736732742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736732742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736732742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736732813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736732923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736732963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736732963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736732963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T05:57:52,993 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:52,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T05:57:52,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:52,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:52,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:53,049 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,052 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166b4796f8d427445ea5c6bcfe3a00c323_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b4796f8d427445ea5c6bcfe3a00c323_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:53,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/b9db884210914faf9d88e6336bf7ac2c, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:53,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/b9db884210914faf9d88e6336bf7ac2c is 175, key is test_row_0/A:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742247_1423 (size=39949) 2024-11-16T05:57:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:53,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736733283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736733283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736733283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736733333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,457 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/b9db884210914faf9d88e6336bf7ac2c 2024-11-16T05:57:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736733443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/a36bd0133ff840898532655578d6dd0e is 50, key is test_row_0/B:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:53,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742248_1424 (size=12301) 2024-11-16T05:57:53,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:53,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736733794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736733803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736733803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:53,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/a36bd0133ff840898532655578d6dd0e 2024-11-16T05:57:53,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/74dc37d680ce40bd893494893848ec26 is 50, key is test_row_0/C:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742249_1425 (size=12301) 2024-11-16T05:57:53,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/74dc37d680ce40bd893494893848ec26 2024-11-16T05:57:53,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/b9db884210914faf9d88e6336bf7ac2c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c 2024-11-16T05:57:53,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c, entries=200, sequenceid=351, filesize=39.0 K 2024-11-16T05:57:53,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/a36bd0133ff840898532655578d6dd0e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e 2024-11-16T05:57:53,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T05:57:53,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/74dc37d680ce40bd893494893848ec26 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26 2024-11-16T05:57:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T05:57:53,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1316ms, sequenceid=351, compaction requested=true 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:53,909 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:53,909 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:53,910 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:53,910 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:53,910 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:53,910 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:53,910 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,910 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,911 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=100.5 K 2024-11-16T05:57:53,911 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,911 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c] 2024-11-16T05:57:53,911 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/af07d626457041a7a6c5cc59620b57c3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=36.5 K 2024-11-16T05:57:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,911 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c29f4e679345aca000f2dacbb04bb4, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:53,911 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting af07d626457041a7a6c5cc59620b57c3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:53,911 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 37824d18233a4669b62bd9bef1ce76c0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731736670868 2024-11-16T05:57:53,912 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0761f1d3d07943cf9c33bb23f53e86d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731736670868 2024-11-16T05:57:53,912 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b9db884210914faf9d88e6336bf7ac2c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672241 2024-11-16T05:57:53,912 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36bd0133ff840898532655578d6dd0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672262 2024-11-16T05:57:53,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T05:57:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:53,914 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T05:57:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:53,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:53,926 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:53,927 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/06636a1c053e45cbbb9a26a542c1a7b2 is 50, key is test_row_0/B:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:53,928 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c44fc8af355749b1994a001fb3129da7_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_1/A:col10/1731736672614/Put/seqid=0 2024-11-16T05:57:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,965 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b3ee7fdab8764b3cadf9a868bf3517fd_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:53,968 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b3ee7fdab8764b3cadf9a868bf3517fd_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:53,968 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b3ee7fdab8764b3cadf9a868bf3517fd_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742250_1426 (size=12881) 2024-11-16T05:57:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742251_1427 (size=9914) 2024-11-16T05:57:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742252_1428 (size=4469) 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,001 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#364 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,002 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/74a2788078d14486ba14247d07d7effc is 175, key is test_row_0/A:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,005 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c44fc8af355749b1994a001fb3129da7_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c44fc8af355749b1994a001fb3129da7_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cf54f90a7abb4b62bbd90c30870d7136, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cf54f90a7abb4b62bbd90c30870d7136 is 175, key is test_row_1/A:col10/1731736672614/Put/seqid=0 2024-11-16T05:57:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742253_1429 (size=31835) 2024-11-16T05:57:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742254_1430 (size=22561) 2024-11-16T05:57:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cf54f90a7abb4b62bbd90c30870d7136 2024-11-16T05:57:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7475d8d7c2484e2194f4373070c039c8 is 50, key is test_row_1/B:col10/1731736672614/Put/seqid=0 2024-11-16T05:57:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742255_1431 (size=9857) 2024-11-16T05:57:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,384 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/06636a1c053e45cbbb9a26a542c1a7b2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/06636a1c053e45cbbb9a26a542c1a7b2 2024-11-16T05:57:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,388 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into 06636a1c053e45cbbb9a26a542c1a7b2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:54,388 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:54,388 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=13, startTime=1731736673909; duration=0sec 2024-11-16T05:57:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,388 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:54,388 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:54,388 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,392 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,392 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:54,392 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:54,392 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/02f4b13784f34ef2a9692f8c568c0d72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=36.5 K 2024-11-16T05:57:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,393 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f4b13784f34ef2a9692f8c568c0d72, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731736669825 2024-11-16T05:57:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,393 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8adcf8e1b564d06bb79e3afa7e4292a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731736670868 2024-11-16T05:57:54,394 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74dc37d680ce40bd893494893848ec26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672262 2024-11-16T05:57:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,402 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#367 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,402 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c88a69e0f9b64364bcb06ebebb04a5e4 is 50, key is test_row_0/C:col10/1731736672263/Put/seqid=0 2024-11-16T05:57:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742256_1432 (size=12881) 2024-11-16T05:57:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:54,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:54,425 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/c88a69e0f9b64364bcb06ebebb04a5e4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c88a69e0f9b64364bcb06ebebb04a5e4 2024-11-16T05:57:54,432 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/74a2788078d14486ba14247d07d7effc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc 2024-11-16T05:57:54,434 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into c88a69e0f9b64364bcb06ebebb04a5e4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:54,434 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:54,434 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=13, startTime=1731736673909; duration=0sec 2024-11-16T05:57:54,435 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:54,435 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:54,437 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into 74a2788078d14486ba14247d07d7effc(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:54,437 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:54,438 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=13, startTime=1731736673909; duration=0sec 2024-11-16T05:57:54,438 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:54,438 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736674459, max timestamp=1731736674500Num uniques 859; 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736674459, max timestamp=1731736674500Num uniques 859; 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736674459, max timestamp=1731736674500Num uniques 859; 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:54,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:57:54,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7475d8d7c2484e2194f4373070c039c8 2024-11-16T05:57:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/60fe2499d83e4dd08ee768ae9851b57d is 50, key is test_row_1/C:col10/1731736672614/Put/seqid=0 2024-11-16T05:57:54,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742257_1433 (size=9857) 2024-11-16T05:57:54,535 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/60fe2499d83e4dd08ee768ae9851b57d 2024-11-16T05:57:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/cf54f90a7abb4b62bbd90c30870d7136 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136 2024-11-16T05:57:54,543 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136, entries=100, sequenceid=359, filesize=22.0 K 2024-11-16T05:57:54,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7475d8d7c2484e2194f4373070c039c8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8 2024-11-16T05:57:54,547 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8, entries=100, sequenceid=359, filesize=9.6 K 2024-11-16T05:57:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/60fe2499d83e4dd08ee768ae9851b57d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d 2024-11-16T05:57:54,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d, entries=100, sequenceid=359, filesize=9.6 K 2024-11-16T05:57:54,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=87.22 KB/89310 for 4b8bf1d7473bdc01de0055e89dea26d3 in 638ms, sequenceid=359, compaction requested=false 2024-11-16T05:57:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T05:57:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T05:57:54,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T05:57:54,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-11-16T05:57:54,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5630 sec 2024-11-16T05:57:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:54,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=93.93 KB heapSize=260.77 KB 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:54,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:57:54,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a86caee79a1d43d9bd5eb55de84c703e_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:54,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742258_1434 (size=14994) 2024-11-16T05:57:54,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736734630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736734640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736734743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736734762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736734812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,820 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736734833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736734833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,833 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,834 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736734963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:54,982 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:54,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a86caee79a1d43d9bd5eb55de84c703e_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a86caee79a1d43d9bd5eb55de84c703e_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:54,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/07af1fb6db694e10b87da30936197cd8, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:54,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/07af1fb6db694e10b87da30936197cd8 is 175, key is test_row_0/A:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736734973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742259_1435 (size=39949) 2024-11-16T05:57:55,001 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/07af1fb6db694e10b87da30936197cd8 2024-11-16T05:57:55,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 is 50, key is test_row_0/B:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742260_1436 (size=12301) 2024-11-16T05:57:55,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 2024-11-16T05:57:55,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/95e1a6a4012f4433a1105be747991297 is 50, key is test_row_0/C:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742261_1437 (size=12301) 2024-11-16T05:57:55,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/95e1a6a4012f4433a1105be747991297 2024-11-16T05:57:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/07af1fb6db694e10b87da30936197cd8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8 2024-11-16T05:57:55,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8, entries=200, sequenceid=392, filesize=39.0 K 2024-11-16T05:57:55,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 2024-11-16T05:57:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:57:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/95e1a6a4012f4433a1105be747991297 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297 2024-11-16T05:57:55,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T05:57:55,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~260.72 KB/266976, currentSize=87.22 KB/89310 for 4b8bf1d7473bdc01de0055e89dea26d3 in 580ms, sequenceid=392, compaction requested=true 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T05:57:55,139 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:55,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:55,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:55,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:55,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c88a69e0f9b64364bcb06ebebb04a5e4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=34.2 K 2024-11-16T05:57:55,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:55,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:55,141 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,141 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=92.1 K 2024-11-16T05:57:55,141 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8] 2024-11-16T05:57:55,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c88a69e0f9b64364bcb06ebebb04a5e4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672262 2024-11-16T05:57:55,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fe2499d83e4dd08ee768ae9851b57d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731736672614 2024-11-16T05:57:55,142 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a2788078d14486ba14247d07d7effc, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672262 2024-11-16T05:57:55,143 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e1a6a4012f4433a1105be747991297, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:55,143 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf54f90a7abb4b62bbd90c30870d7136, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731736672614 2024-11-16T05:57:55,144 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07af1fb6db694e10b87da30936197cd8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:55,165 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:55,165 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/2ef26ff3eb8b40c9b8a2ce369383385c is 50, key is test_row_0/C:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T05:57:55,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:55,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T05:57:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:55,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:55,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:55,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:55,181 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:55,206 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116c74c4b66406e484db5041ca0bab16fba_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:55,208 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116c74c4b66406e484db5041ca0bab16fba_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:55,208 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c74c4b66406e484db5041ca0bab16fba_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:55,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742262_1438 (size=12983) 2024-11-16T05:57:55,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742263_1439 (size=4469) 2024-11-16T05:57:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:55,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:55,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:55,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111664e00d4a4d7e415b8839221c0938fd8f_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736674628/Put/seqid=0 2024-11-16T05:57:55,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742264_1440 (size=14994) 2024-11-16T05:57:55,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:55,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111664e00d4a4d7e415b8839221c0938fd8f_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111664e00d4a4d7e415b8839221c0938fd8f_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:55,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/276b733710b44bca856143f44367d5a5, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:55,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/276b733710b44bca856143f44367d5a5 is 175, key is test_row_0/A:col10/1731736674628/Put/seqid=0 2024-11-16T05:57:55,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742265_1441 (size=39949) 2024-11-16T05:57:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736735366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736735376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736735483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736735492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,634 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/2ef26ff3eb8b40c9b8a2ce369383385c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2ef26ff3eb8b40c9b8a2ce369383385c 2024-11-16T05:57:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,642 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into 2ef26ff3eb8b40c9b8a2ce369383385c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:55,642 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:55,642 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=13, startTime=1731736675139; duration=0sec 2024-11-16T05:57:55,642 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:55,642 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:55,643 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:57:55,643 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:57:55,643 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:55,643 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,644 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/06636a1c053e45cbbb9a26a542c1a7b2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=34.2 K 2024-11-16T05:57:55,644 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 06636a1c053e45cbbb9a26a542c1a7b2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731736672262 2024-11-16T05:57:55,644 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 7475d8d7c2484e2194f4373070c039c8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731736672614 2024-11-16T05:57:55,644 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb86fe7ce4c4dada0e66d1c7f348bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:55,648 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#373 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:55,648 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/d1c9f8cee8d44cfeb76ac06a49466fe5 is 175, key is test_row_0/A:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:55,675 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:55,675 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/b8c488adb5fa412482016424a8948ad4 is 50, key is test_row_0/B:col10/1731736674557/Put/seqid=0 2024-11-16T05:57:55,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742266_1442 (size=31937) 2024-11-16T05:57:55,699 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/d1c9f8cee8d44cfeb76ac06a49466fe5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5 2024-11-16T05:57:55,703 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into d1c9f8cee8d44cfeb76ac06a49466fe5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:55,703 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:55,703 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=13, startTime=1731736675139; duration=0sec 2024-11-16T05:57:55,703 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:55,703 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736735703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742267_1443 (size=12983) 2024-11-16T05:57:55,723 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/b8c488adb5fa412482016424a8948ad4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/b8c488adb5fa412482016424a8948ad4 2024-11-16T05:57:55,731 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into b8c488adb5fa412482016424a8948ad4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:55,731 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:55,731 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=13, startTime=1731736675139; duration=0sec 2024-11-16T05:57:55,731 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:55,731 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:55,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736735713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:55,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/276b733710b44bca856143f44367d5a5 2024-11-16T05:57:55,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/de27d1fb610543a28c7a77b637c7c424 is 50, key is test_row_0/B:col10/1731736674628/Put/seqid=0 2024-11-16T05:57:55,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:55,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742268_1444 (size=12301) 2024-11-16T05:57:55,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/de27d1fb610543a28c7a77b637c7c424 2024-11-16T05:57:55,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5ab03f09645f468087770d2f05e5275c is 50, key is test_row_0/C:col10/1731736674628/Put/seqid=0 2024-11-16T05:57:55,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742269_1445 (size=12301) 2024-11-16T05:57:55,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:55,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736736024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736736043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:56,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5ab03f09645f468087770d2f05e5275c 2024-11-16T05:57:56,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:56,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:56,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:56,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:56,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:56,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/276b733710b44bca856143f44367d5a5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5 2024-11-16T05:57:56,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5, entries=200, sequenceid=409, filesize=39.0 K 2024-11-16T05:57:56,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/de27d1fb610543a28c7a77b637c7c424 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424 2024-11-16T05:57:56,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T05:57:56,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/5ab03f09645f468087770d2f05e5275c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c 2024-11-16T05:57:56,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T05:57:56,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4b8bf1d7473bdc01de0055e89dea26d3 in 999ms, sequenceid=409, compaction requested=false 2024-11-16T05:57:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:56,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:56,403 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:56,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116266d5e0a486549bfaafd7c5a7c049100_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736675372/Put/seqid=0 2024-11-16T05:57:56,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742270_1446 (size=12454) 2024-11-16T05:57:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:56,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116266d5e0a486549bfaafd7c5a7c049100_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116266d5e0a486549bfaafd7c5a7c049100_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/da691e57fad74f25ace214c5210da4d8, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/da691e57fad74f25ace214c5210da4d8 is 175, key is test_row_0/A:col10/1731736675372/Put/seqid=0 2024-11-16T05:57:56,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742271_1447 (size=31255) 2024-11-16T05:57:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:56,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736736633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736736633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736736743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736736753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37066 deadline: 1731736736823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,832 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37082 deadline: 1731736736843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,853 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37104 deadline: 1731736736853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,874 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in 4b8bf1d7473bdc01de0055e89dea26d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:57:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,930 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/da691e57fad74f25ace214c5210da4d8 2024-11-16T05:57:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/faa25081075e430a89ef342262d4328b is 50, key is test_row_0/B:col10/1731736675372/Put/seqid=0 2024-11-16T05:57:56,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736736963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:56,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736736973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742272_1448 (size=12301) 2024-11-16T05:57:57,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:57:57,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736737283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736737292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:57,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/faa25081075e430a89ef342262d4328b 2024-11-16T05:57:57,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/69bfe08df4ef4f7590d7c490b5fdd655 is 50, key is test_row_0/C:col10/1731736675372/Put/seqid=0 2024-11-16T05:57:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742273_1449 (size=12301) 2024-11-16T05:57:57,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/69bfe08df4ef4f7590d7c490b5fdd655 2024-11-16T05:57:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/da691e57fad74f25ace214c5210da4d8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8 2024-11-16T05:57:57,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8, entries=150, sequenceid=427, filesize=30.5 K 2024-11-16T05:57:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/faa25081075e430a89ef342262d4328b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b 2024-11-16T05:57:57,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T05:57:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/69bfe08df4ef4f7590d7c490b5fdd655 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655 2024-11-16T05:57:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T05:57:57,511 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1107ms, sequenceid=427, compaction requested=true 2024-11-16T05:57:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T05:57:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T05:57:57,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T05:57:57,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-11-16T05:57:57,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3500 sec 2024-11-16T05:57:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:57,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:57,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111630e33f1b00c945eba6d61fb257e7c5c2_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:57,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742274_1450 (size=14994) 2024-11-16T05:57:57,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736737866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736737879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736737983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736737992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736738203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736738203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:58,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111630e33f1b00c945eba6d61fb257e7c5c2_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111630e33f1b00c945eba6d61fb257e7c5c2_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:58,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/88b57000f0c34ab68ddb82296b9230be, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/88b57000f0c34ab68ddb82296b9230be is 175, key is test_row_0/A:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742275_1451 (size=39949) 2024-11-16T05:57:58,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=446, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/88b57000f0c34ab68ddb82296b9230be 2024-11-16T05:57:58,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/462e6ba924c540bfa82e0845d504717b is 50, key is test_row_0/B:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742276_1452 (size=12301) 2024-11-16T05:57:58,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/462e6ba924c540bfa82e0845d504717b 2024-11-16T05:57:58,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/aebf3c1eba8047ba8519f123c8428e42 is 50, key is test_row_0/C:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742277_1453 (size=12301) 2024-11-16T05:57:58,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/aebf3c1eba8047ba8519f123c8428e42 2024-11-16T05:57:58,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/88b57000f0c34ab68ddb82296b9230be as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be 2024-11-16T05:57:58,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be, entries=200, sequenceid=446, filesize=39.0 K 2024-11-16T05:57:58,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/462e6ba924c540bfa82e0845d504717b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b 2024-11-16T05:57:58,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b, entries=150, sequenceid=446, filesize=12.0 K 2024-11-16T05:57:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/aebf3c1eba8047ba8519f123c8428e42 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42 2024-11-16T05:57:58,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42, entries=150, sequenceid=446, filesize=12.0 K 2024-11-16T05:57:58,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4b8bf1d7473bdc01de0055e89dea26d3 in 517ms, sequenceid=446, compaction requested=true 2024-11-16T05:57:58,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:58,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:57:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:58,325 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:58,325 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:57:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b8bf1d7473bdc01de0055e89dea26d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:57:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:58,325 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:58,325 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/B is initiating minor compaction (all files) 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/A is initiating minor compaction (all files) 2024-11-16T05:57:58,326 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/B in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:58,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/A in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:58,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=139.7 K 2024-11-16T05:57:58,326 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/b8c488adb5fa412482016424a8948ad4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=48.7 K 2024-11-16T05:57:58,326 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be] 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b8c488adb5fa412482016424a8948ad4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1c9f8cee8d44cfeb76ac06a49466fe5, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 276b733710b44bca856143f44367d5a5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731736674620 2024-11-16T05:57:58,326 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting de27d1fb610543a28c7a77b637c7c424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731736674620 2024-11-16T05:57:58,327 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting da691e57fad74f25ace214c5210da4d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731736675351 2024-11-16T05:57:58,327 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting faa25081075e430a89ef342262d4328b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731736675351 2024-11-16T05:57:58,327 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b57000f0c34ab68ddb82296b9230be, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731736676612 2024-11-16T05:57:58,327 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 462e6ba924c540bfa82e0845d504717b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731736676612 2024-11-16T05:57:58,348 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,354 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:58,355 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/9296f7d8d41d4f19adc9d433fde36239 is 50, key is test_row_0/B:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,356 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d21f70f7fae640939da4a76274869f12_4b8bf1d7473bdc01de0055e89dea26d3 store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,359 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d21f70f7fae640939da4a76274869f12_4b8bf1d7473bdc01de0055e89dea26d3, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,359 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d21f70f7fae640939da4a76274869f12_4b8bf1d7473bdc01de0055e89dea26d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742279_1455 (size=4469) 2024-11-16T05:57:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742278_1454 (size=13119) 2024-11-16T05:57:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:58,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:58,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:58,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116255ba80894514f169dcdac19d59fde03_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736677835/Put/seqid=0 2024-11-16T05:57:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742280_1456 (size=14994) 2024-11-16T05:57:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736738607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736738612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736738722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736738733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,784 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#A#compaction#384 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:58,784 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/ebc385d382cb49f99e170643043caa7d is 175, key is test_row_0/A:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,790 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/9296f7d8d41d4f19adc9d433fde36239 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9296f7d8d41d4f19adc9d433fde36239 2024-11-16T05:57:58,798 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/B of 4b8bf1d7473bdc01de0055e89dea26d3 into 9296f7d8d41d4f19adc9d433fde36239(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:58,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:58,798 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/B, priority=12, startTime=1731736678325; duration=0sec 2024-11-16T05:57:58,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:57:58,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:B 2024-11-16T05:57:58,798 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:57:58,799 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:57:58,799 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): 4b8bf1d7473bdc01de0055e89dea26d3/C is initiating minor compaction (all files) 2024-11-16T05:57:58,799 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b8bf1d7473bdc01de0055e89dea26d3/C in TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:58,799 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2ef26ff3eb8b40c9b8a2ce369383385c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp, totalSize=48.7 K 2024-11-16T05:57:58,799 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef26ff3eb8b40c9b8a2ce369383385c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731736674513 2024-11-16T05:57:58,800 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab03f09645f468087770d2f05e5275c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731736674620 2024-11-16T05:57:58,800 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 69bfe08df4ef4f7590d7c490b5fdd655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731736675351 2024-11-16T05:57:58,800 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting aebf3c1eba8047ba8519f123c8428e42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731736676612 2024-11-16T05:57:58,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742281_1457 (size=32073) 2024-11-16T05:57:58,804 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/ebc385d382cb49f99e170643043caa7d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/ebc385d382cb49f99e170643043caa7d 2024-11-16T05:57:58,808 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/A of 4b8bf1d7473bdc01de0055e89dea26d3 into ebc385d382cb49f99e170643043caa7d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:58,808 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:58,808 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/A, priority=12, startTime=1731736678324; duration=0sec 2024-11-16T05:57:58,808 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:58,808 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:A 2024-11-16T05:57:58,811 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b8bf1d7473bdc01de0055e89dea26d3#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:57:58,812 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/d4eb210650e94ec7b3f19b7f48d5c0f5 is 50, key is test_row_0/C:col10/1731736676615/Put/seqid=0 2024-11-16T05:57:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742282_1458 (size=13119) 2024-11-16T05:57:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736738943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:57:58,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116255ba80894514f169dcdac19d59fde03_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116255ba80894514f169dcdac19d59fde03_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:57:58,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/9429bfb36ba141348ec48edeea605fcb, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:57:58,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/9429bfb36ba141348ec48edeea605fcb is 175, key is test_row_0/A:col10/1731736677835/Put/seqid=0 2024-11-16T05:57:58,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736738953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:58,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742283_1459 (size=39949) 2024-11-16T05:57:58,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=460, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/9429bfb36ba141348ec48edeea605fcb 2024-11-16T05:57:58,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/61ea657dada84da5a9c4c8a38c4d6b4e is 50, key is test_row_0/B:col10/1731736677835/Put/seqid=0 2024-11-16T05:57:58,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742284_1460 (size=12301) 2024-11-16T05:57:59,219 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/d4eb210650e94ec7b3f19b7f48d5c0f5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d4eb210650e94ec7b3f19b7f48d5c0f5 2024-11-16T05:57:59,228 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b8bf1d7473bdc01de0055e89dea26d3/C of 4b8bf1d7473bdc01de0055e89dea26d3 into d4eb210650e94ec7b3f19b7f48d5c0f5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:57:59,228 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:59,228 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., storeName=4b8bf1d7473bdc01de0055e89dea26d3/C, priority=12, startTime=1731736678325; duration=0sec 2024-11-16T05:57:59,228 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:57:59,228 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b8bf1d7473bdc01de0055e89dea26d3:C 2024-11-16T05:57:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736739262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736739273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T05:57:59,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:57:59,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:57:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T05:57:59,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:57:59,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:57:59,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:57:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:57:59,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/61ea657dada84da5a9c4c8a38c4d6b4e 2024-11-16T05:57:59,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b96e04ffe69e4d59851307af902ec510 is 50, key is test_row_0/C:col10/1731736677835/Put/seqid=0 2024-11-16T05:57:59,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742285_1461 (size=12301) 2024-11-16T05:57:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:57:59,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T05:57:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T05:57:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:59,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:57:59,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37088 deadline: 1731736739783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:59,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:57:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731736739792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:57:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:57:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:57:59,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T05:57:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. as already flushing 2024-11-16T05:57:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b96e04ffe69e4d59851307af902ec510 2024-11-16T05:57:59,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:57:59,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/9429bfb36ba141348ec48edeea605fcb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/9429bfb36ba141348ec48edeea605fcb 2024-11-16T05:57:59,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/9429bfb36ba141348ec48edeea605fcb, entries=200, sequenceid=460, filesize=39.0 K 2024-11-16T05:57:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/61ea657dada84da5a9c4c8a38c4d6b4e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/61ea657dada84da5a9c4c8a38c4d6b4e 2024-11-16T05:57:59,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/61ea657dada84da5a9c4c8a38c4d6b4e, entries=150, sequenceid=460, filesize=12.0 K 2024-11-16T05:57:59,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/b96e04ffe69e4d59851307af902ec510 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b96e04ffe69e4d59851307af902ec510 2024-11-16T05:57:59,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b96e04ffe69e4d59851307af902ec510, entries=150, sequenceid=460, filesize=12.0 K 2024-11-16T05:57:59,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1314ms, sequenceid=460, compaction requested=false 2024-11-16T05:57:59,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:57:59,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T05:57:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:57:59,974 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:57:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:57:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:57:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111667bdd0bbf1264abd9a1c76fa1d6cccdc_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736678605/Put/seqid=0 2024-11-16T05:57:59,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742286_1462 (size=12454) 2024-11-16T05:57:59,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:00,008 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111667bdd0bbf1264abd9a1c76fa1d6cccdc_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111667bdd0bbf1264abd9a1c76fa1d6cccdc_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/46a2f233a98e4e5aad1352f790973f7d, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:58:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/46a2f233a98e4e5aad1352f790973f7d is 175, key is test_row_0/A:col10/1731736678605/Put/seqid=0 2024-11-16T05:58:00,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742287_1463 (size=31255) 2024-11-16T05:58:00,025 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=481, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/46a2f233a98e4e5aad1352f790973f7d 2024-11-16T05:58:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/150c869b3c29423eb923c8acd1989a00 is 50, key is test_row_0/B:col10/1731736678605/Put/seqid=0 2024-11-16T05:58:00,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742288_1464 (size=12301) 2024-11-16T05:58:00,044 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/150c869b3c29423eb923c8acd1989a00 2024-11-16T05:58:00,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/840771ca8f3746a3bf6edd158b3afe32 is 50, key is test_row_0/C:col10/1731736678605/Put/seqid=0 2024-11-16T05:58:00,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742289_1465 (size=12301) 2024-11-16T05:58:00,215 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-11-16T05:58:00,215 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,215 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-16T05:58:00,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,215 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,215 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-16T05:58:00,215 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,215 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,222 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-11-16T05:58:00,222 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,222 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,222 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,224 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-16T05:58:00,224 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,224 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,224 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,476 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/840771ca8f3746a3bf6edd158b3afe32 2024-11-16T05:58:00,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/46a2f233a98e4e5aad1352f790973f7d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/46a2f233a98e4e5aad1352f790973f7d 2024-11-16T05:58:00,482 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/46a2f233a98e4e5aad1352f790973f7d, entries=150, sequenceid=481, filesize=30.5 K 2024-11-16T05:58:00,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/150c869b3c29423eb923c8acd1989a00 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/150c869b3c29423eb923c8acd1989a00 2024-11-16T05:58:00,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/150c869b3c29423eb923c8acd1989a00, entries=150, sequenceid=481, filesize=12.0 K 2024-11-16T05:58:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/840771ca8f3746a3bf6edd158b3afe32 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/840771ca8f3746a3bf6edd158b3afe32 2024-11-16T05:58:00,490 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/840771ca8f3746a3bf6edd158b3afe32, entries=150, sequenceid=481, filesize=12.0 K 2024-11-16T05:58:00,491 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 4b8bf1d7473bdc01de0055e89dea26d3 in 518ms, sequenceid=481, compaction requested=true 2024-11-16T05:58:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4b8bf1d7473bdc01de0055e89dea26d3: 2024-11-16T05:58:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:58:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T05:58:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T05:58:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:58:00,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T05:58:00,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-16T05:58:00,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1460 sec 2024-11-16T05:58:00,805 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-16T05:58:00,805 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-16T05:58:00,805 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,805 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,805 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,805 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,805 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,805 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,855 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-16T05:58:00,855 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,855 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,855 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,874 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-16T05:58:00,875 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,875 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,875 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:00,895 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-16T05:58:00,895 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:00,895 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,895 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:00,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:01,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be] to archive 2024-11-16T05:58:01,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:01,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a7b729f2288342809fcbe1a8189a7764 2024-11-16T05:58:01,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/76be090ef7aa4671bdc08f997125382e 2024-11-16T05:58:01,496 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/236e67039f4a4b02ba93f1fab97110b1 2024-11-16T05:58:01,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6ffc0eb60ca8499592b796e1d8ba9278 2024-11-16T05:58:01,498 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a230683e376f4f468a48af9ab88a7a13 2024-11-16T05:58:01,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/f89ce9ab119a454995f91613c0b18a38 2024-11-16T05:58:01,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/2cf8a3fa29334499a5c920a94cabf6f3 2024-11-16T05:58:01,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c20cc09797e94e7e882ff6fb0e064045 2024-11-16T05:58:01,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c63dbb2007984e16855cf982a8226e97 2024-11-16T05:58:01,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/6edaf956fcf340a4b85d163ef0d328b9 2024-11-16T05:58:01,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c8c770cef06a4022955ef38214a10884 2024-11-16T05:58:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T05:58:01,503 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 870 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2529 rows 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 843 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2528 rows 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 841 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2522 rows 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 873 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2542 rows 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 852 2024-11-16T05:58:01,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2554 rows 2024-11-16T05:58:01,504 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:58:01,504 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:58:01,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:01,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:01,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:01,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:58:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:01,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cafd2de0b6f7462c9506228e6042de72 2024-11-16T05:58:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:01,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/4f49057631094ee1a4173819af8ee127 2024-11-16T05:58:01,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736681509"}]},"ts":"1731736681509"} 2024-11-16T05:58:01,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/e36260f9cb5b48dfaff89707c751afe4 2024-11-16T05:58:01,510 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:58:01,510 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:58:01,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/a8c29f4e679345aca000f2dacbb04bb4 2024-11-16T05:58:01,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:58:01,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/37824d18233a4669b62bd9bef1ce76c0 2024-11-16T05:58:01,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, UNASSIGN}] 2024-11-16T05:58:01,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/b9db884210914faf9d88e6336bf7ac2c 2024-11-16T05:58:01,514 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, UNASSIGN 2024-11-16T05:58:01,514 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:01,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/74a2788078d14486ba14247d07d7effc 2024-11-16T05:58:01,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, UNASSIGN because future has completed 2024-11-16T05:58:01,516 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:58:01,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:01,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/cf54f90a7abb4b62bbd90c30870d7136 2024-11-16T05:58:01,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/07af1fb6db694e10b87da30936197cd8 2024-11-16T05:58:01,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/d1c9f8cee8d44cfeb76ac06a49466fe5 2024-11-16T05:58:01,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/276b733710b44bca856143f44367d5a5 2024-11-16T05:58:01,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/da691e57fad74f25ace214c5210da4d8 2024-11-16T05:58:01,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/88b57000f0c34ab68ddb82296b9230be 2024-11-16T05:58:01,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/104cfa235eae41d0a7d6317e7b7172c4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ec3f5a4bd3724437b43db409c48b2d11, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/dcb99aca07184cdfbac042b9a9b8c0e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/af07d626457041a7a6c5cc59620b57c3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/06636a1c053e45cbbb9a26a542c1a7b2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/b8c488adb5fa412482016424a8948ad4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b] to archive 2024-11-16T05:58:01,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:01,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/74eefa9a9986415d8a9cfd1d5b171add 2024-11-16T05:58:01,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/89981394e9994503812f62468cb55d5c 2024-11-16T05:58:01,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7e78b601f4cf4f5997b4e112b53268c0 2024-11-16T05:58:01,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/104cfa235eae41d0a7d6317e7b7172c4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/104cfa235eae41d0a7d6317e7b7172c4 2024-11-16T05:58:01,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/1fa7fe31dc1a4cad92bb4c66dd5a00d4 2024-11-16T05:58:01,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/409a768ff31f4ea2918c25311540e8aa 2024-11-16T05:58:01,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ec3f5a4bd3724437b43db409c48b2d11 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ec3f5a4bd3724437b43db409c48b2d11 2024-11-16T05:58:01,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9cbf120ddbd7425b9509617a323532ec 2024-11-16T05:58:01,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/6a6659d742ba4bb7bc1980088f78847d 2024-11-16T05:58:01,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/dcb99aca07184cdfbac042b9a9b8c0e9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/dcb99aca07184cdfbac042b9a9b8c0e9 2024-11-16T05:58:01,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7f77827f690c41dda74a9bd35bde75dc 2024-11-16T05:58:01,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/412941966c6046618003df0b7900d6be 2024-11-16T05:58:01,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/ca4e60098696497e9819eb3cc90c7c2a 2024-11-16T05:58:01,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/af07d626457041a7a6c5cc59620b57c3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/af07d626457041a7a6c5cc59620b57c3 2024-11-16T05:58:01,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/f0891260f719451f94453c15791f29b0 2024-11-16T05:58:01,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/0761f1d3d07943cf9c33bb23f53e86d0 2024-11-16T05:58:01,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/06636a1c053e45cbbb9a26a542c1a7b2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/06636a1c053e45cbbb9a26a542c1a7b2 2024-11-16T05:58:01,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/a36bd0133ff840898532655578d6dd0e 2024-11-16T05:58:01,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7475d8d7c2484e2194f4373070c039c8 2024-11-16T05:58:01,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/b8c488adb5fa412482016424a8948ad4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/b8c488adb5fa412482016424a8948ad4 2024-11-16T05:58:01,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/4bb86fe7ce4c4dada0e66d1c7f348bd3 2024-11-16T05:58:01,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/de27d1fb610543a28c7a77b637c7c424 2024-11-16T05:58:01,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/faa25081075e430a89ef342262d4328b 2024-11-16T05:58:01,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/462e6ba924c540bfa82e0845d504717b 2024-11-16T05:58:01,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c2546821bd9f400e94ba485d6cc1d76e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/01005995bb47404d9226268acd40811b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e74d7cddb62b4788a660910ec310149b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/02f4b13784f34ef2a9692f8c568c0d72, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c88a69e0f9b64364bcb06ebebb04a5e4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2ef26ff3eb8b40c9b8a2ce369383385c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42] to archive 2024-11-16T05:58:01,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:01,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d86cae383f514239a796fd347acf192a 2024-11-16T05:58:01,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e3db32787a004aa88c164294905e85e7 2024-11-16T05:58:01,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1864ac1fd00f41589337e939f96d8c87 2024-11-16T05:58:01,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c2546821bd9f400e94ba485d6cc1d76e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c2546821bd9f400e94ba485d6cc1d76e 2024-11-16T05:58:01,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/f05f7c68b47342ad905e4100e68bf352 2024-11-16T05:58:01,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b242f3ac6d134c11b16dfd9de2814212 2024-11-16T05:58:01,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/01005995bb47404d9226268acd40811b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/01005995bb47404d9226268acd40811b 2024-11-16T05:58:01,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/1d17eceed8874eec9818ed61db95f843 2024-11-16T05:58:01,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5d258ff907db492183ab37af339d3110 2024-11-16T05:58:01,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e74d7cddb62b4788a660910ec310149b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/e74d7cddb62b4788a660910ec310149b 2024-11-16T05:58:01,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2f051d54c8fd42df904027ba0dfb2df0 2024-11-16T05:58:01,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/55501db6dd7d4d6a8c3e6c6a6c1e8f20 2024-11-16T05:58:01,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c22f69ffe4f84fc9acb97814cc995c3f 2024-11-16T05:58:01,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/02f4b13784f34ef2a9692f8c568c0d72 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/02f4b13784f34ef2a9692f8c568c0d72 2024-11-16T05:58:01,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b39b2a941e59484499f02781536b1127 2024-11-16T05:58:01,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/a8adcf8e1b564d06bb79e3afa7e4292a 2024-11-16T05:58:01,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c88a69e0f9b64364bcb06ebebb04a5e4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/c88a69e0f9b64364bcb06ebebb04a5e4 2024-11-16T05:58:01,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/74dc37d680ce40bd893494893848ec26 2024-11-16T05:58:01,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/60fe2499d83e4dd08ee768ae9851b57d 2024-11-16T05:58:01,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2ef26ff3eb8b40c9b8a2ce369383385c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/2ef26ff3eb8b40c9b8a2ce369383385c 2024-11-16T05:58:01,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/95e1a6a4012f4433a1105be747991297 2024-11-16T05:58:01,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/5ab03f09645f468087770d2f05e5275c 2024-11-16T05:58:01,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/69bfe08df4ef4f7590d7c490b5fdd655 2024-11-16T05:58:01,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/aebf3c1eba8047ba8519f123c8428e42 2024-11-16T05:58:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:01,669 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 4b8bf1d7473bdc01de0055e89dea26d3, disabling compactions & flushes 2024-11-16T05:58:01,669 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. after waiting 0 ms 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:58:01,669 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 4b8bf1d7473bdc01de0055e89dea26d3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=A 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=B 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b8bf1d7473bdc01de0055e89dea26d3, store=C 2024-11-16T05:58:01,669 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:01,674 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116985ee1283d5c45eba9c1196f8d419c45_4b8bf1d7473bdc01de0055e89dea26d3 is 50, key is test_row_0/A:col10/1731736680893/Put/seqid=0 2024-11-16T05:58:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742290_1466 (size=12454) 2024-11-16T05:58:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:02,079 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:02,086 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116985ee1283d5c45eba9c1196f8d419c45_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116985ee1283d5c45eba9c1196f8d419c45_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:02,088 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c086b45420bc4915a0b72f43d6df1751, store: [table=TestAcidGuarantees family=A region=4b8bf1d7473bdc01de0055e89dea26d3] 2024-11-16T05:58:02,089 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c086b45420bc4915a0b72f43d6df1751 is 175, key is test_row_0/A:col10/1731736680893/Put/seqid=0 2024-11-16T05:58:02,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742291_1467 (size=31255) 2024-11-16T05:58:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:02,493 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c086b45420bc4915a0b72f43d6df1751 2024-11-16T05:58:02,499 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7b2b996450bb4768820df618df7833f7 is 50, key is test_row_0/B:col10/1731736680893/Put/seqid=0 2024-11-16T05:58:02,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742292_1468 (size=12301) 2024-11-16T05:58:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:02,903 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7b2b996450bb4768820df618df7833f7 2024-11-16T05:58:02,909 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/254067f38dfc48cfbfeebe87f89ee33b is 50, key is test_row_0/C:col10/1731736680893/Put/seqid=0 2024-11-16T05:58:02,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742293_1469 (size=12301) 2024-11-16T05:58:03,334 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/254067f38dfc48cfbfeebe87f89ee33b 2024-11-16T05:58:03,338 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/A/c086b45420bc4915a0b72f43d6df1751 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c086b45420bc4915a0b72f43d6df1751 2024-11-16T05:58:03,341 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c086b45420bc4915a0b72f43d6df1751, entries=150, sequenceid=489, filesize=30.5 K 2024-11-16T05:58:03,342 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/B/7b2b996450bb4768820df618df7833f7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7b2b996450bb4768820df618df7833f7 2024-11-16T05:58:03,345 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7b2b996450bb4768820df618df7833f7, entries=150, sequenceid=489, filesize=12.0 K 2024-11-16T05:58:03,346 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/.tmp/C/254067f38dfc48cfbfeebe87f89ee33b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/254067f38dfc48cfbfeebe87f89ee33b 2024-11-16T05:58:03,350 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/254067f38dfc48cfbfeebe87f89ee33b, entries=150, sequenceid=489, filesize=12.0 K 2024-11-16T05:58:03,351 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1682ms, sequenceid=489, compaction requested=true 2024-11-16T05:58:03,356 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits/492.seqid, newMaxSeqId=492, maxSeqId=4 2024-11-16T05:58:03,356 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. 2024-11-16T05:58:03,356 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 4b8bf1d7473bdc01de0055e89dea26d3: Waiting for close lock at 1731736681669Running coprocessor pre-close hooks at 1731736681669Disabling compacts and flushes for region at 1731736681669Disabling writes for close at 1731736681669Obtaining lock to block concurrent updates at 1731736681669Preparing flush snapshotting stores in 4b8bf1d7473bdc01de0055e89dea26d3 at 1731736681669Finished memstore snapshotting TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731736681669Flushing stores of TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3. at 1731736681670 (+1 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/A: creating writer at 1731736681670Flushing mob file 4b8bf1d7473bdc01de0055e89dea26d3/A: appending metadata at 1731736681674 (+4 ms)Flushing mob file 4b8bf1d7473bdc01de0055e89dea26d3/A: closing flushed file at 1731736681674Flushing 4b8bf1d7473bdc01de0055e89dea26d3/A: appending metadata at 1731736682088 (+414 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/A: closing flushed file at 1731736682088Flushing 4b8bf1d7473bdc01de0055e89dea26d3/B: creating writer at 1731736682497 (+409 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/B: appending metadata at 1731736682499 (+2 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/B: closing flushed file at 1731736682499Flushing 4b8bf1d7473bdc01de0055e89dea26d3/C: creating writer at 1731736682906 (+407 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/C: appending metadata at 1731736682908 (+2 ms)Flushing 4b8bf1d7473bdc01de0055e89dea26d3/C: closing flushed file at 1731736682908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30b1198c: reopening flushed file at 1731736683337 (+429 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cbada76: reopening flushed file at 1731736683341 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62a0837e: reopening flushed file at 1731736683345 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4b8bf1d7473bdc01de0055e89dea26d3 in 1682ms, sequenceid=489, compaction requested=true at 1731736683351 (+6 ms)Writing region close event to WAL at 1731736683352 (+1 ms)Running coprocessor post-close hooks at 1731736683356 (+4 ms)Closed at 1731736683356 2024-11-16T05:58:03,358 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4b8bf1d7473bdc01de0055e89dea26d3, regionState=CLOSED 2024-11-16T05:58:03,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:03,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T05:58:03,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b8bf1d7473bdc01de0055e89dea26d3, server=3456ee6a3164,45735,1731736558702 in 1.8440 sec 2024-11-16T05:58:03,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-16T05:58:03,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b8bf1d7473bdc01de0055e89dea26d3, UNASSIGN in 1.8480 sec 2024-11-16T05:58:03,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T05:58:03,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8520 sec 2024-11-16T05:58:03,365 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736683365"}]},"ts":"1731736683365"} 2024-11-16T05:58:03,366 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:58:03,366 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:58:03,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-16T05:58:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T05:58:03,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:03,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:58:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:58:03,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,657 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,659 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits] 2024-11-16T05:58:03,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/46a2f233a98e4e5aad1352f790973f7d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/46a2f233a98e4e5aad1352f790973f7d 2024-11-16T05:58:03,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/9429bfb36ba141348ec48edeea605fcb to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/9429bfb36ba141348ec48edeea605fcb 2024-11-16T05:58:03,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c086b45420bc4915a0b72f43d6df1751 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/c086b45420bc4915a0b72f43d6df1751 2024-11-16T05:58:03,663 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/ebc385d382cb49f99e170643043caa7d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/A/ebc385d382cb49f99e170643043caa7d 2024-11-16T05:58:03,665 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/150c869b3c29423eb923c8acd1989a00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/150c869b3c29423eb923c8acd1989a00 2024-11-16T05:58:03,665 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/61ea657dada84da5a9c4c8a38c4d6b4e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/61ea657dada84da5a9c4c8a38c4d6b4e 2024-11-16T05:58:03,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7b2b996450bb4768820df618df7833f7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/7b2b996450bb4768820df618df7833f7 2024-11-16T05:58:03,667 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9296f7d8d41d4f19adc9d433fde36239 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/B/9296f7d8d41d4f19adc9d433fde36239 2024-11-16T05:58:03,669 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/254067f38dfc48cfbfeebe87f89ee33b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/254067f38dfc48cfbfeebe87f89ee33b 2024-11-16T05:58:03,670 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/840771ca8f3746a3bf6edd158b3afe32 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/840771ca8f3746a3bf6edd158b3afe32 2024-11-16T05:58:03,670 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b96e04ffe69e4d59851307af902ec510 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/b96e04ffe69e4d59851307af902ec510 2024-11-16T05:58:03,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d4eb210650e94ec7b3f19b7f48d5c0f5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/C/d4eb210650e94ec7b3f19b7f48d5c0f5 2024-11-16T05:58:03,673 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits/492.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3/recovered.edits/492.seqid 2024-11-16T05:58:03,673 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,674 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:58:03,674 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:58:03,675 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:58:03,677 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116036d0bc951164951a7a27809001b6b60_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116036d0bc951164951a7a27809001b6b60_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,678 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a2a2c26af84a70bd7195f1c0208128_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111611a2a2c26af84a70bd7195f1c0208128_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,679 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111614ce6be019c540d2bdb5f7f53f2dd9e1_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111614ce6be019c540d2bdb5f7f53f2dd9e1_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,680 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116168ba7738f494f0492a896692172b8c9_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116168ba7738f494f0492a896692172b8c9_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,680 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116255ba80894514f169dcdac19d59fde03_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116255ba80894514f169dcdac19d59fde03_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,681 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116266d5e0a486549bfaafd7c5a7c049100_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116266d5e0a486549bfaafd7c5a7c049100_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,682 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296e2cfdb67c421fab9e9847418b8e6b_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296e2cfdb67c421fab9e9847418b8e6b_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,683 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111630e33f1b00c945eba6d61fb257e7c5c2_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111630e33f1b00c945eba6d61fb257e7c5c2_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,683 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163855ce48f463436093c0c0ae7270a450_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163855ce48f463436093c0c0ae7270a450_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,684 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116590dad7a516442779f4438bf5f6669a5_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116590dad7a516442779f4438bf5f6669a5_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,685 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111664e00d4a4d7e415b8839221c0938fd8f_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111664e00d4a4d7e415b8839221c0938fd8f_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,686 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111667bdd0bbf1264abd9a1c76fa1d6cccdc_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111667bdd0bbf1264abd9a1c76fa1d6cccdc_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,686 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b4796f8d427445ea5c6bcfe3a00c323_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b4796f8d427445ea5c6bcfe3a00c323_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,687 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adc49efa7d9429ca3d765eb1977dcc9_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adc49efa7d9429ca3d765eb1977dcc9_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,688 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116985ee1283d5c45eba9c1196f8d419c45_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116985ee1283d5c45eba9c1196f8d419c45_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,688 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169d6c35e63089436682629cb0d3072584_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169d6c35e63089436682629cb0d3072584_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,689 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a86caee79a1d43d9bd5eb55de84c703e_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a86caee79a1d43d9bd5eb55de84c703e_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,690 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c44fc8af355749b1994a001fb3129da7_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c44fc8af355749b1994a001fb3129da7_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,691 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c454f1e7127548d18c01c8f2151263d6_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c454f1e7127548d18c01c8f2151263d6_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,691 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5fd597fea704340ac50a6dc613f0120_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5fd597fea704340ac50a6dc613f0120_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,692 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d42eb6f51bf8433a9883ee6ab2e21bcb_4b8bf1d7473bdc01de0055e89dea26d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d42eb6f51bf8433a9883ee6ab2e21bcb_4b8bf1d7473bdc01de0055e89dea26d3 2024-11-16T05:58:03,692 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:58:03,694 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,696 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:58:03,698 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:58:03,699 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,699 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:58:03,699 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736683699"}]},"ts":"9223372036854775807"} 2024-11-16T05:58:03,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:58:03,701 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4b8bf1d7473bdc01de0055e89dea26d3, NAME => 'TestAcidGuarantees,,1731736658027.4b8bf1d7473bdc01de0055e89dea26d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:58:03,701 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:58:03,701 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736683701"}]},"ts":"9223372036854775807"} 2024-11-16T05:58:03,702 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:58:03,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-16T05:58:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T05:58:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:58:03,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:03,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-62023640_22 at /127.0.0.1:58124 [Waiting for operation #953] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1574838696_22 at /127.0.0.1:39828 [Waiting for operation #1332] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-62023640_22 at /127.0.0.1:60442 [Waiting for operation #642] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1574838696_22 at /127.0.0.1:60436 [Waiting for operation #672] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=539 (was 480) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3388 (was 2714) - AvailableMemoryMB LEAK? - 2024-11-16T05:58:03,779 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=539, ProcessCount=11, AvailableMemoryMB=3388 2024-11-16T05:58:03,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:58:03,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:58:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:03,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:58:03,782 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:03,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-16T05:58:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:03,783 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:58:03,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742294_1470 (size=1061) 2024-11-16T05:58:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:04,193 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:58:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742295_1471 (size=53) 2024-11-16T05:58:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:04,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:58:04,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c2babb3b1cd0035fe3947dd2fdeb4457, disabling compactions & flushes 2024-11-16T05:58:04,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. after waiting 0 ms 2024-11-16T05:58:04,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,606 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2babb3b1cd0035fe3947dd2fdeb4457: Waiting for close lock at 1731736684605Disabling compacts and flushes for region at 1731736684605Disabling writes for close at 1731736684605Writing region close event to WAL at 1731736684605Closed at 1731736684605 2024-11-16T05:58:04,608 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:58:04,609 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736684608"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736684608"}]},"ts":"1731736684608"} 2024-11-16T05:58:04,613 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:58:04,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:58:04,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736684615"}]},"ts":"1731736684615"} 2024-11-16T05:58:04,618 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:58:04,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, ASSIGN}] 2024-11-16T05:58:04,620 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, ASSIGN 2024-11-16T05:58:04,620 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:58:04,772 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c2babb3b1cd0035fe3947dd2fdeb4457, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:04,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, ASSIGN because future has completed 2024-11-16T05:58:04,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:04,936 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,936 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:58:04,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:58:04,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,939 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,940 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:04,941 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2babb3b1cd0035fe3947dd2fdeb4457 columnFamilyName A 2024-11-16T05:58:04,941 DEBUG [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:04,941 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(327): Store=c2babb3b1cd0035fe3947dd2fdeb4457/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:04,941 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,943 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:04,943 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2babb3b1cd0035fe3947dd2fdeb4457 columnFamilyName B 2024-11-16T05:58:04,943 DEBUG [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:04,944 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(327): Store=c2babb3b1cd0035fe3947dd2fdeb4457/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:04,944 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,946 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:04,946 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2babb3b1cd0035fe3947dd2fdeb4457 columnFamilyName C 2024-11-16T05:58:04,946 DEBUG [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:04,947 INFO [StoreOpener-c2babb3b1cd0035fe3947dd2fdeb4457-1 {}] regionserver.HStore(327): Store=c2babb3b1cd0035fe3947dd2fdeb4457/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:04,947 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,947 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,948 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,949 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,950 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,950 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,951 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:58:04,953 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,956 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:58:04,957 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened c2babb3b1cd0035fe3947dd2fdeb4457; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74076802, jitterRate=0.10383036732673645}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:58:04,957 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:04,958 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for c2babb3b1cd0035fe3947dd2fdeb4457: Running coprocessor pre-open hook at 1731736684937Writing region info on filesystem at 1731736684937Initializing all the Stores at 1731736684938 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736684938Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736684938Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736684938Cleaning up temporary data from old regions at 1731736684950 (+12 ms)Running coprocessor post-open hooks at 1731736684957 (+7 ms)Region opened successfully at 1731736684958 (+1 ms) 2024-11-16T05:58:04,959 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., pid=112, masterSystemTime=1731736684929 2024-11-16T05:58:04,961 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,961 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:04,961 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c2babb3b1cd0035fe3947dd2fdeb4457, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:04,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:04,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T05:58:04,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 in 189 msec 2024-11-16T05:58:04,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-16T05:58:04,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, ASSIGN in 347 msec 2024-11-16T05:58:04,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:58:04,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736684968"}]},"ts":"1731736684968"} 2024-11-16T05:58:04,970 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:58:04,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:58:04,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1910 sec 2024-11-16T05:58:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-16T05:58:05,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:05,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e6c6b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2953059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,953 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,956 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:58:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:58:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0b2c31 2024-11-16T05:58:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:58:05,958 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:58:05,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a975164, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4273bdd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24791a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,966 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d22c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,968 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb631b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0c5ee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,973 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e0ff1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1a7b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,977 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c437b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb01272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607d6b90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa9b827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3f7a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b05e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5d768b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22db4810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be5e7fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ed47b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:05,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:05,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17158ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:05,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b5635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:05,995 DEBUG [Thread-2104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,995 DEBUG [Thread-2111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,995 DEBUG [Thread-2108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,996 DEBUG [Thread-2115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,996 DEBUG [Thread-2117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,996 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,996 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,998 DEBUG [Thread-2106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:05,999 DEBUG [Thread-2119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:05,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:05,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:05,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:05,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:06,000 DEBUG [Thread-2113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-16T05:58:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:06,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:06,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:06,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:06,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2] 2024-11-16T05:58:06,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:06,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/40100f98d670415b9cbe0f8a9cf378a0 is 50, key is test_row_0/A:col10/1731736686010/Put/seqid=0 2024-11-16T05:58:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742296_1472 (size=14341) 2024-11-16T05:58:06,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/40100f98d670415b9cbe0f8a9cf378a0 2024-11-16T05:58:06,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736686020, max timestamp=1731736686050Num uniques 859; 2024-11-16T05:58:06,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:06,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736686020, max timestamp=1731736686050Num uniques 859; 2024-11-16T05:58:06,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736686020, max timestamp=1731736686050Num uniques 859; 2024-11-16T05:58:06,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:06,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:06,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/25119416db12443f8120eeacf3cdde7a is 50, key is test_row_0/B:col10/1731736686010/Put/seqid=0 2024-11-16T05:58:06,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742297_1473 (size=12001) 2024-11-16T05:58:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736746077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736746079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736746080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736746081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736746087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:06,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736746203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736746203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736746203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736746203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736746203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736746413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736746413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736746413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736746413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736746413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/25119416db12443f8120eeacf3cdde7a 2024-11-16T05:58:06,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/f5e1595770b54ba3b31f72c3d32eba28 is 50, key is test_row_0/C:col10/1731736686010/Put/seqid=0 2024-11-16T05:58:06,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742298_1474 (size=12001) 2024-11-16T05:58:06,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736746733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736746733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736746733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736746733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736746733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:06,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:06,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/f5e1595770b54ba3b31f72c3d32eba28 2024-11-16T05:58:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/40100f98d670415b9cbe0f8a9cf378a0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0 2024-11-16T05:58:06,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0, entries=200, sequenceid=11, filesize=14.0 K 2024-11-16T05:58:06,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/25119416db12443f8120eeacf3cdde7a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a 2024-11-16T05:58:06,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T05:58:06,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/f5e1595770b54ba3b31f72c3d32eba28 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28 2024-11-16T05:58:06,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T05:58:06,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c2babb3b1cd0035fe3947dd2fdeb4457 in 895ms, sequenceid=11, compaction requested=false 2024-11-16T05:58:06,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:06,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T05:58:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:06,928 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T05:58:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/756cce77292d4620a554c98aa6c923ce is 50, key is test_row_0/A:col10/1731736686075/Put/seqid=0 2024-11-16T05:58:06,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742299_1475 (size=12001) 2024-11-16T05:58:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:07,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:07,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736747258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736747261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736747262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736747262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736747263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,337 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/756cce77292d4620a554c98aa6c923ce 2024-11-16T05:58:07,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/66574114d01d4d7bb1093e3ed7b67ebf is 50, key is test_row_0/B:col10/1731736686075/Put/seqid=0 2024-11-16T05:58:07,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742300_1476 (size=12001) 2024-11-16T05:58:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736747593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736747593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736747593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736747593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736747593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,748 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/66574114d01d4d7bb1093e3ed7b67ebf 2024-11-16T05:58:07,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/677fce07180f478796bc42f0d4c16aa9 is 50, key is test_row_0/C:col10/1731736686075/Put/seqid=0 2024-11-16T05:58:07,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742301_1477 (size=12001) 2024-11-16T05:58:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736747903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736747903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736747913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736747913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736747913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:08,162 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/677fce07180f478796bc42f0d4c16aa9 2024-11-16T05:58:08,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/756cce77292d4620a554c98aa6c923ce as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce 2024-11-16T05:58:08,169 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:58:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/66574114d01d4d7bb1093e3ed7b67ebf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf 2024-11-16T05:58:08,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:58:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/677fce07180f478796bc42f0d4c16aa9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9 2024-11-16T05:58:08,177 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T05:58:08,177 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1249ms, sequenceid=46, compaction requested=false 2024-11-16T05:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-16T05:58:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-16T05:58:08,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T05:58:08,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-11-16T05:58:08,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.1820 sec 2024-11-16T05:58:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:08,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:58:08,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:08,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:08,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/282695aaa2ae4ce5834ed7f0e0487cfc is 50, key is test_row_0/A:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742302_1478 (size=14341) 2024-11-16T05:58:08,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/282695aaa2ae4ce5834ed7f0e0487cfc 2024-11-16T05:58:08,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/3a67b70044b4400ab1f9a08a93f56f7d is 50, key is test_row_0/B:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742303_1479 (size=12001) 2024-11-16T05:58:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736748454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736748455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731736688436, max timestamp=1731736688464Num uniques 859; 2024-11-16T05:58:08,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731736688436, max timestamp=1731736688464Num uniques 859; 2024-11-16T05:58:08,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731736688436, max timestamp=1731736688464Num uniques 859; 2024-11-16T05:58:08,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T05:58:08,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T05:58:08,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736748496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736748503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736748505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736748573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736748573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736748612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736748612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736748622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736748793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736748793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736748822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736748822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/3a67b70044b4400ab1f9a08a93f56f7d 2024-11-16T05:58:08,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b282430f10e842c4bfb5a9b3003236fa is 50, key is test_row_0/C:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:08,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736748843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:08,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742304_1480 (size=12001) 2024-11-16T05:58:08,877 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:58:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736749113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736749113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736749133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736749133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736749163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b282430f10e842c4bfb5a9b3003236fa 2024-11-16T05:58:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/282695aaa2ae4ce5834ed7f0e0487cfc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc 2024-11-16T05:58:09,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc, entries=200, sequenceid=57, filesize=14.0 K 2024-11-16T05:58:09,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/3a67b70044b4400ab1f9a08a93f56f7d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d 2024-11-16T05:58:09,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d, entries=150, sequenceid=57, filesize=11.7 K 2024-11-16T05:58:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b282430f10e842c4bfb5a9b3003236fa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa 2024-11-16T05:58:09,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa, entries=150, sequenceid=57, filesize=11.7 K 2024-11-16T05:58:09,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c2babb3b1cd0035fe3947dd2fdeb4457 in 849ms, sequenceid=57, compaction requested=true 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:09,276 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:09,276 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:09,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:09,277 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:09,277 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=39.7 K 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:09,277 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:09,277 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.2 K 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40100f98d670415b9cbe0f8a9cf378a0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731736686003 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756cce77292d4620a554c98aa6c923ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736686075 2024-11-16T05:58:09,277 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 25119416db12443f8120eeacf3cdde7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731736686003 2024-11-16T05:58:09,278 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282695aaa2ae4ce5834ed7f0e0487cfc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687251 2024-11-16T05:58:09,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 66574114d01d4d7bb1093e3ed7b67ebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736686075 2024-11-16T05:58:09,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a67b70044b4400ab1f9a08a93f56f7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687261 2024-11-16T05:58:09,309 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:09,310 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d9e81a12220d43ad9bfb2ae0db9499b2 is 50, key is test_row_0/A:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:09,320 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:09,320 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9482ad85997647f2b6f4721bb8280f27 is 50, key is test_row_0/B:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:09,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742305_1481 (size=12104) 2024-11-16T05:58:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742306_1482 (size=12104) 2024-11-16T05:58:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:09,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:09,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30fdaab070704bbc88a197ec95c78b60 is 50, key is test_row_0/A:col10/1731736689628/Put/seqid=0 2024-11-16T05:58:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742307_1483 (size=14341) 2024-11-16T05:58:09,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736749649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30fdaab070704bbc88a197ec95c78b60 2024-11-16T05:58:09,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/cbaa7ad01d9d429b952d5b3d582d04da is 50, key is test_row_0/B:col10/1731736689628/Put/seqid=0 2024-11-16T05:58:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736749650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736749651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742308_1484 (size=12001) 2024-11-16T05:58:09,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/cbaa7ad01d9d429b952d5b3d582d04da 2024-11-16T05:58:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736749652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b4d5395b35364c8783f8a39207eef489 is 50, key is test_row_0/C:col10/1731736689628/Put/seqid=0 2024-11-16T05:58:09,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736749673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742309_1485 (size=12001) 2024-11-16T05:58:09,739 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d9e81a12220d43ad9bfb2ae0db9499b2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d9e81a12220d43ad9bfb2ae0db9499b2 2024-11-16T05:58:09,743 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into d9e81a12220d43ad9bfb2ae0db9499b2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:09,743 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:09,743 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736689276; duration=0sec 2024-11-16T05:58:09,743 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:09,743 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:09,743 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:09,745 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:09,745 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:09,745 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:09,745 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.2 K 2024-11-16T05:58:09,747 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e1595770b54ba3b31f72c3d32eba28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731736686003 2024-11-16T05:58:09,748 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677fce07180f478796bc42f0d4c16aa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731736686075 2024-11-16T05:58:09,748 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b282430f10e842c4bfb5a9b3003236fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687261 2024-11-16T05:58:09,751 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9482ad85997647f2b6f4721bb8280f27 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9482ad85997647f2b6f4721bb8280f27 2024-11-16T05:58:09,754 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 9482ad85997647f2b6f4721bb8280f27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:09,755 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:09,755 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736689276; duration=0sec 2024-11-16T05:58:09,755 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:09,755 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:09,756 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:09,757 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/fe9fdbbe09fa41aea2f1f3f211eacab5 is 50, key is test_row_0/C:col10/1731736687262/Put/seqid=0 2024-11-16T05:58:09,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742310_1486 (size=12104) 2024-11-16T05:58:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736749763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736749763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736749763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736749782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736749983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736749983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736749983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736749992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b4d5395b35364c8783f8a39207eef489 2024-11-16T05:58:10,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30fdaab070704bbc88a197ec95c78b60 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60 2024-11-16T05:58:10,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60, entries=200, sequenceid=91, filesize=14.0 K 2024-11-16T05:58:10,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/cbaa7ad01d9d429b952d5b3d582d04da as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da 2024-11-16T05:58:10,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T05:58:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/b4d5395b35364c8783f8a39207eef489 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489 2024-11-16T05:58:10,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T05:58:10,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for c2babb3b1cd0035fe3947dd2fdeb4457 in 479ms, sequenceid=91, compaction requested=false 2024-11-16T05:58:10,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T05:58:10,163 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:10,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-16T05:58:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:10,165 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:10,165 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:10,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:10,174 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/fe9fdbbe09fa41aea2f1f3f211eacab5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fe9fdbbe09fa41aea2f1f3f211eacab5 2024-11-16T05:58:10,178 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into fe9fdbbe09fa41aea2f1f3f211eacab5(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:10,178 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:10,178 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736689276; duration=0sec 2024-11-16T05:58:10,178 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:10,178 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:10,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:10,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/e92d826d321649a68ee59920f1c5ca62 is 50, key is test_row_0/A:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:10,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742311_1487 (size=14341) 2024-11-16T05:58:10,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/e92d826d321649a68ee59920f1c5ca62 2024-11-16T05:58:10,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9707a3fbf941429d87bdd0cffdf5f703 is 50, key is test_row_0/B:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:10,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742312_1488 (size=12001) 2024-11-16T05:58:10,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:10,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:10,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736750341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736690334, max timestamp=1731736690350Num uniques 859; 2024-11-16T05:58:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:10,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736690334, max timestamp=1731736690350Num uniques 859; 2024-11-16T05:58:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736750349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731736690334, max timestamp=1731736690350Num uniques 859; 2024-11-16T05:58:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736750412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736750413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736750462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736750463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736750532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736750532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736750673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736750673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736750683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,693 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9707a3fbf941429d87bdd0cffdf5f703 2024-11-16T05:58:10,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/3ae88e4a23014d15a879b613fe2ed5d8 is 50, key is test_row_0/C:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:10,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742313_1489 (size=12001) 2024-11-16T05:58:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736750742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736750743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:10,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:10,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:10,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736750993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736750993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736751053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736751063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:11,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/3ae88e4a23014d15a879b613fe2ed5d8 2024-11-16T05:58:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/e92d826d321649a68ee59920f1c5ca62 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62 2024-11-16T05:58:11,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62, entries=200, sequenceid=105, filesize=14.0 K 2024-11-16T05:58:11,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9707a3fbf941429d87bdd0cffdf5f703 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703 2024-11-16T05:58:11,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703, entries=150, sequenceid=105, filesize=11.7 K 2024-11-16T05:58:11,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/3ae88e4a23014d15a879b613fe2ed5d8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8 2024-11-16T05:58:11,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8, entries=150, sequenceid=105, filesize=11.7 K 2024-11-16T05:58:11,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c2babb3b1cd0035fe3947dd2fdeb4457 in 841ms, sequenceid=105, compaction requested=true 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:11,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:11,139 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:11,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:11,140 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,140 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d9e81a12220d43ad9bfb2ae0db9499b2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=39.8 K 2024-11-16T05:58:11,140 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9482ad85997647f2b6f4721bb8280f27, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.3 K 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e81a12220d43ad9bfb2ae0db9499b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687261 2024-11-16T05:58:11,140 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9482ad85997647f2b6f4721bb8280f27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687261 2024-11-16T05:58:11,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 30fdaab070704bbc88a197ec95c78b60, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731736688482 2024-11-16T05:58:11,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbaa7ad01d9d429b952d5b3d582d04da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731736688482 2024-11-16T05:58:11,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e92d826d321649a68ee59920f1c5ca62, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:11,141 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9707a3fbf941429d87bdd0cffdf5f703, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:11,147 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:11,147 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6cd8d9e4af964eb3beebf48ea7904fd9 is 50, key is test_row_0/B:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:11,148 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:11,149 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/a9bf151d7a224fff87ee2decdeb5c869 is 50, key is test_row_0/A:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:11,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742314_1490 (size=12207) 2024-11-16T05:58:11,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742315_1491 (size=12207) 2024-11-16T05:58:11,181 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/a9bf151d7a224fff87ee2decdeb5c869 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/a9bf151d7a224fff87ee2decdeb5c869 2024-11-16T05:58:11,188 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into a9bf151d7a224fff87ee2decdeb5c869(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:11,188 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:11,188 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736691139; duration=0sec 2024-11-16T05:58:11,188 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:11,188 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:11,188 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:11,188 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:11,189 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:11,189 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,189 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fe9fdbbe09fa41aea2f1f3f211eacab5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.3 K 2024-11-16T05:58:11,192 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9fdbbe09fa41aea2f1f3f211eacab5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731736687261 2024-11-16T05:58:11,193 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d5395b35364c8783f8a39207eef489, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731736688482 2024-11-16T05:58:11,193 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae88e4a23014d15a879b613fe2ed5d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:11,205 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:11,205 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/262660dbee8b48c08ed5fb44317466e6 is 50, key is test_row_0/C:col10/1731736689651/Put/seqid=0 2024-11-16T05:58:11,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742316_1492 (size=12207) 2024-11-16T05:58:11,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:11,232 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:11,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/9dd7ca1b87674c46a957aed20dca6073 is 50, key is test_row_0/A:col10/1731736690407/Put/seqid=0 2024-11-16T05:58:11,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742317_1493 (size=12151) 2024-11-16T05:58:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:11,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:11,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736751538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736751548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,567 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6cd8d9e4af964eb3beebf48ea7904fd9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6cd8d9e4af964eb3beebf48ea7904fd9 2024-11-16T05:58:11,571 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 6cd8d9e4af964eb3beebf48ea7904fd9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:11,571 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:11,571 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736691139; duration=0sec 2024-11-16T05:58:11,571 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:11,571 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736751573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736751582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,612 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/262660dbee8b48c08ed5fb44317466e6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/262660dbee8b48c08ed5fb44317466e6 2024-11-16T05:58:11,616 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 262660dbee8b48c08ed5fb44317466e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:11,616 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:11,616 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736691139; duration=0sec 2024-11-16T05:58:11,616 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:11,616 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:11,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/9dd7ca1b87674c46a957aed20dca6073 2024-11-16T05:58:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/de21b3aa41cb411a9388cf037cc07286 is 50, key is test_row_0/B:col10/1731736690407/Put/seqid=0 2024-11-16T05:58:11,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742318_1494 (size=12151) 2024-11-16T05:58:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736751653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736751663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736751873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:11,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736751883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,050 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/de21b3aa41cb411a9388cf037cc07286 2024-11-16T05:58:12,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d6f7d22e508f4272bd6c754688f23e42 is 50, key is test_row_0/C:col10/1731736690407/Put/seqid=0 2024-11-16T05:58:12,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742319_1495 (size=12151) 2024-11-16T05:58:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736752182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736752202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:12,458 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d6f7d22e508f4272bd6c754688f23e42 2024-11-16T05:58:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/9dd7ca1b87674c46a957aed20dca6073 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073 2024-11-16T05:58:12,464 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073, entries=150, sequenceid=140, filesize=11.9 K 2024-11-16T05:58:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/de21b3aa41cb411a9388cf037cc07286 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286 2024-11-16T05:58:12,468 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286, entries=150, sequenceid=140, filesize=11.9 K 2024-11-16T05:58:12,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d6f7d22e508f4272bd6c754688f23e42 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42 2024-11-16T05:58:12,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42, entries=150, sequenceid=140, filesize=11.9 K 2024-11-16T05:58:12,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1239ms, sequenceid=140, compaction requested=false 2024-11-16T05:58:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-16T05:58:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-16T05:58:12,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T05:58:12,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-11-16T05:58:12,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.3120 sec 2024-11-16T05:58:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:12,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:58:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:12,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/adcd0504f0a9445bb267b738ca01c10a is 50, key is test_row_0/A:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:12,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742320_1496 (size=14541) 2024-11-16T05:58:12,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736752643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736752648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736752692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736752713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,714 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736752723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736752752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736752762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736752963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736752973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:12,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/adcd0504f0a9445bb267b738ca01c10a 2024-11-16T05:58:13,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/4d2cd2af3ff34021a7503920efce4bc2 is 50, key is test_row_0/B:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:13,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742321_1497 (size=12151) 2024-11-16T05:58:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736753272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736753283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/4d2cd2af3ff34021a7503920efce4bc2 2024-11-16T05:58:13,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/998ea5f26109471a833b072875d4d28c is 50, key is test_row_0/C:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:13,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742322_1498 (size=12151) 2024-11-16T05:58:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736753713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,719 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736753743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,746 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736753782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736753792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:13,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/998ea5f26109471a833b072875d4d28c 2024-11-16T05:58:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/adcd0504f0a9445bb267b738ca01c10a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a 2024-11-16T05:58:13,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a, entries=200, sequenceid=154, filesize=14.2 K 2024-11-16T05:58:13,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/4d2cd2af3ff34021a7503920efce4bc2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2 2024-11-16T05:58:13,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-16T05:58:13,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/998ea5f26109471a833b072875d4d28c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c 2024-11-16T05:58:13,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-16T05:58:13,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1237ms, sequenceid=154, compaction requested=true 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:13,826 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:13,826 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:13,826 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:13,827 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/a9bf151d7a224fff87ee2decdeb5c869, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=38.0 K 2024-11-16T05:58:13,827 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6cd8d9e4af964eb3beebf48ea7904fd9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.7 K 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9bf151d7a224fff87ee2decdeb5c869, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd8d9e4af964eb3beebf48ea7904fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting de21b3aa41cb411a9388cf037cc07286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731736690394 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd7ca1b87674c46a957aed20dca6073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731736690394 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d2cd2af3ff34021a7503920efce4bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:13,827 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting adcd0504f0a9445bb267b738ca01c10a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:13,835 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:13,836 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/50cc3425ed474a8dbe3b8b7fbf0ec98a is 50, key is test_row_0/A:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:13,843 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:13,843 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/3af04221f98548e6bff0b84e4e58a314 is 50, key is test_row_0/B:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:13,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742323_1499 (size=12459) 2024-11-16T05:58:13,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742324_1500 (size=12459) 2024-11-16T05:58:13,852 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/3af04221f98548e6bff0b84e4e58a314 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3af04221f98548e6bff0b84e4e58a314 2024-11-16T05:58:13,855 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 3af04221f98548e6bff0b84e4e58a314(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:13,856 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736693826; duration=0sec 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:13,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:13,856 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:13,857 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/262660dbee8b48c08ed5fb44317466e6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.7 K 2024-11-16T05:58:13,857 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 262660dbee8b48c08ed5fb44317466e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731736689648 2024-11-16T05:58:13,857 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f7d22e508f4272bd6c754688f23e42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731736690394 2024-11-16T05:58:13,857 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 998ea5f26109471a833b072875d4d28c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:13,862 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:13,862 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2eea3345def24f6aae0de5d46720cc50 is 50, key is test_row_0/C:col10/1731736691525/Put/seqid=0 2024-11-16T05:58:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742325_1501 (size=12459) 2024-11-16T05:58:14,252 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/50cc3425ed474a8dbe3b8b7fbf0ec98a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/50cc3425ed474a8dbe3b8b7fbf0ec98a 2024-11-16T05:58:14,255 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into 50cc3425ed474a8dbe3b8b7fbf0ec98a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:14,255 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:14,255 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736693826; duration=0sec 2024-11-16T05:58:14,255 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:14,255 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:14,275 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2eea3345def24f6aae0de5d46720cc50 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2eea3345def24f6aae0de5d46720cc50 2024-11-16T05:58:14,278 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 2eea3345def24f6aae0de5d46720cc50(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:14,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:14,278 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736693826; duration=0sec 2024-11-16T05:58:14,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:14,278 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T05:58:14,343 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:14,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T05:58:14,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:58:14,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:14,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:58:14,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T05:58:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:14,497 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/716d26fab5e848fdaaa8b4c79f9cca87 is 50, key is test_row_0/A:col10/1731736692646/Put/seqid=0 2024-11-16T05:58:14,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742326_1502 (size=12151) 2024-11-16T05:58:14,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/716d26fab5e848fdaaa8b4c79f9cca87 2024-11-16T05:58:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/345db1287701464b8a958f722895ebdf is 50, key is test_row_0/B:col10/1731736692646/Put/seqid=0 2024-11-16T05:58:14,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742327_1503 (size=12151) 2024-11-16T05:58:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:58:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:14,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736754840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:14,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736754841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:14,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/345db1287701464b8a958f722895ebdf 2024-11-16T05:58:14,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d314ce585bf84c3abe2bfcc798705d70 is 50, key is test_row_0/C:col10/1731736692646/Put/seqid=0 2024-11-16T05:58:14,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742328_1504 (size=12151) 2024-11-16T05:58:14,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736754952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:14,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736754952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:58:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736755163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736755163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,352 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d314ce585bf84c3abe2bfcc798705d70 2024-11-16T05:58:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/716d26fab5e848fdaaa8b4c79f9cca87 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87 2024-11-16T05:58:15,359 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T05:58:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/345db1287701464b8a958f722895ebdf as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf 2024-11-16T05:58:15,363 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T05:58:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/d314ce585bf84c3abe2bfcc798705d70 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70 2024-11-16T05:58:15,367 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T05:58:15,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c2babb3b1cd0035fe3947dd2fdeb4457 in 871ms, sequenceid=177, compaction requested=false 2024-11-16T05:58:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T05:58:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T05:58:15,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T05:58:15,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-16T05:58:15,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0300 sec 2024-11-16T05:58:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/ecb7a8b30cc84065bc002df98a4817e5 is 50, key is test_row_0/A:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T05:58:15,483 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:15,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T05:58:15,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:15,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:15,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742329_1505 (size=14541) 2024-11-16T05:58:15,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:15,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/ecb7a8b30cc84065bc002df98a4817e5 2024-11-16T05:58:15,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d6684e7daf4a4a249af62e950e5e5afb is 50, key is test_row_0/B:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:15,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742330_1506 (size=12151) 2024-11-16T05:58:15,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d6684e7daf4a4a249af62e950e5e5afb 2024-11-16T05:58:15,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/68f8f77699da4e1591e78ca887734944 is 50, key is test_row_0/C:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:15,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742331_1507 (size=12151) 2024-11-16T05:58:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736755539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736755544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:15,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T05:58:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736755652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736755652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736755733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,742 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736755752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,758 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T05:58:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736755863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736755863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:15,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/68f8f77699da4e1591e78ca887734944 2024-11-16T05:58:15,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/ecb7a8b30cc84065bc002df98a4817e5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5 2024-11-16T05:58:15,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5, entries=200, sequenceid=190, filesize=14.2 K 2024-11-16T05:58:15,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d6684e7daf4a4a249af62e950e5e5afb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb 2024-11-16T05:58:15,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb, entries=150, sequenceid=190, filesize=11.9 K 2024-11-16T05:58:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/68f8f77699da4e1591e78ca887734944 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944 2024-11-16T05:58:15,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944, entries=150, sequenceid=190, filesize=11.9 K 2024-11-16T05:58:15,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c2babb3b1cd0035fe3947dd2fdeb4457 in 442ms, sequenceid=190, compaction requested=true 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:15,920 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:15,920 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:15,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:15,921 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,921 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,921 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/50cc3425ed474a8dbe3b8b7fbf0ec98a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=38.2 K 2024-11-16T05:58:15,921 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3af04221f98548e6bff0b84e4e58a314, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.9 K 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50cc3425ed474a8dbe3b8b7fbf0ec98a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:15,921 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af04221f98548e6bff0b84e4e58a314, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:15,922 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716d26fab5e848fdaaa8b4c79f9cca87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731736692631 2024-11-16T05:58:15,922 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 345db1287701464b8a958f722895ebdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731736692631 2024-11-16T05:58:15,922 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb7a8b30cc84065bc002df98a4817e5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:15,922 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d6684e7daf4a4a249af62e950e5e5afb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:15,928 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:15,928 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6d59d2a8224440d889f9e7914ae53e38 is 50, key is test_row_0/B:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:15,928 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:15,929 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/e04ad4c1483a4876a13bb2b534f889c8 is 50, key is test_row_0/A:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:15,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742333_1509 (size=12561) 2024-11-16T05:58:15,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742332_1508 (size=12561) 2024-11-16T05:58:15,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:15,944 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/7ea00a5443454b069015bf6d0f4ab022 is 50, key is test_row_0/A:col10/1731736695538/Put/seqid=0 2024-11-16T05:58:15,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742334_1510 (size=12151) 2024-11-16T05:58:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:16,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736756216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736756217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,339 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6d59d2a8224440d889f9e7914ae53e38 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6d59d2a8224440d889f9e7914ae53e38 2024-11-16T05:58:16,339 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/e04ad4c1483a4876a13bb2b534f889c8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e04ad4c1483a4876a13bb2b534f889c8 2024-11-16T05:58:16,343 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 6d59d2a8224440d889f9e7914ae53e38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:16,343 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into e04ad4c1483a4876a13bb2b534f889c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,343 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736695920; duration=0sec 2024-11-16T05:58:16,343 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736695920; duration=0sec 2024-11-16T05:58:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736756333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:16,343 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736756333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:16,344 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:16,344 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2eea3345def24f6aae0de5d46720cc50, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=35.9 K 2024-11-16T05:58:16,344 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eea3345def24f6aae0de5d46720cc50, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731736691525 2024-11-16T05:58:16,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d314ce585bf84c3abe2bfcc798705d70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731736692631 2024-11-16T05:58:16,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f8f77699da4e1591e78ca887734944, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:16,349 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:16,350 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/6d10385408f549eca4e5b7b3775a71a9 is 50, key is test_row_0/C:col10/1731736695476/Put/seqid=0 2024-11-16T05:58:16,351 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/7ea00a5443454b069015bf6d0f4ab022 2024-11-16T05:58:16,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742335_1511 (size=12561) 2024-11-16T05:58:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/60b36e602fb04cf1b9e75ff836acf44f is 50, key is test_row_0/B:col10/1731736695538/Put/seqid=0 2024-11-16T05:58:16,362 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/6d10385408f549eca4e5b7b3775a71a9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/6d10385408f549eca4e5b7b3775a71a9 2024-11-16T05:58:16,365 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 6d10385408f549eca4e5b7b3775a71a9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:16,366 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:16,366 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736695920; duration=0sec 2024-11-16T05:58:16,366 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:16,366 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:16,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742336_1512 (size=12151) 2024-11-16T05:58:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736756552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736756552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54522 deadline: 1731736756733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,741 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,768 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/60b36e602fb04cf1b9e75ff836acf44f 2024-11-16T05:58:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/24382b04b6e644618eb14ccb3c1736a2 is 50, key is test_row_0/C:col10/1731736695538/Put/seqid=0 2024-11-16T05:58:16,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742337_1513 (size=12151) 2024-11-16T05:58:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736756873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736756873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/24382b04b6e644618eb14ccb3c1736a2 2024-11-16T05:58:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/7ea00a5443454b069015bf6d0f4ab022 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022 2024-11-16T05:58:17,185 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T05:58:17,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/60b36e602fb04cf1b9e75ff836acf44f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f 2024-11-16T05:58:17,188 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T05:58:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/24382b04b6e644618eb14ccb3c1736a2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2 2024-11-16T05:58:17,192 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T05:58:17,192 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1248ms, sequenceid=209, compaction requested=false 2024-11-16T05:58:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T05:58:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T05:58:17,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T05:58:17,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-11-16T05:58:17,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7110 sec 2024-11-16T05:58:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:17,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:17,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/8ad96f316fc14b8baeedc572f390fbb4 is 50, key is test_row_0/A:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:17,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742338_1514 (size=14541) 2024-11-16T05:58:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736757445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736757451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736757562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736757563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T05:58:17,633 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:17,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T05:58:17,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T05:58:17,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:17,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T05:58:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736757772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736757772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:17,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T05:58:17,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:17,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:17,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:17,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/8ad96f316fc14b8baeedc572f390fbb4 2024-11-16T05:58:17,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/65d777b6d3b44228a42ca8c93878db0a is 50, key is test_row_0/B:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:17,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742339_1515 (size=12151) 2024-11-16T05:58:17,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/65d777b6d3b44228a42ca8c93878db0a 2024-11-16T05:58:17,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/41ce6d868b914f779e9d26f7e33c5df9 is 50, key is test_row_0/C:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:17,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742340_1516 (size=12151) 2024-11-16T05:58:17,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T05:58:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:17,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T05:58:18,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T05:58:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736758093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736758093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/41ce6d868b914f779e9d26f7e33c5df9 2024-11-16T05:58:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/8ad96f316fc14b8baeedc572f390fbb4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4 2024-11-16T05:58:18,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4, entries=200, sequenceid=226, filesize=14.2 K 2024-11-16T05:58:18,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/65d777b6d3b44228a42ca8c93878db0a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a 2024-11-16T05:58:18,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a, entries=150, sequenceid=226, filesize=11.9 K 2024-11-16T05:58:18,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/41ce6d868b914f779e9d26f7e33c5df9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9 2024-11-16T05:58:18,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9, entries=150, sequenceid=226, filesize=11.9 K 2024-11-16T05:58:18,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c2babb3b1cd0035fe3947dd2fdeb4457 in 844ms, sequenceid=226, compaction requested=true 2024-11-16T05:58:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:18,232 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:18,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:18,232 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:18,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:18,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:18,234 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:18,234 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:18,234 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:18,234 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:18,234 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,235 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e04ad4c1483a4876a13bb2b534f889c8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=38.3 K 2024-11-16T05:58:18,235 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,235 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6d59d2a8224440d889f9e7914ae53e38, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.0 K 2024-11-16T05:58:18,235 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d59d2a8224440d889f9e7914ae53e38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:18,235 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04ad4c1483a4876a13bb2b534f889c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:18,235 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea00a5443454b069015bf6d0f4ab022, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731736695533 2024-11-16T05:58:18,235 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b36e602fb04cf1b9e75ff836acf44f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731736695533 2024-11-16T05:58:18,236 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad96f316fc14b8baeedc572f390fbb4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:18,236 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d777b6d3b44228a42ca8c93878db0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:18,242 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:18,242 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:18,242 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d380c212b58c4141a3b7be86feb91530 is 50, key is test_row_0/A:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:18,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T05:58:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,247 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:58:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,248 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/0797e79802f8403d9425803238ce6545 is 50, key is test_row_0/B:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/93ada95e5147446bb1688f8e76b5582f is 50, key is test_row_0/A:col10/1731736697444/Put/seqid=0 2024-11-16T05:58:18,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742341_1517 (size=12663) 2024-11-16T05:58:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742342_1518 (size=12663) 2024-11-16T05:58:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T05:58:18,264 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d380c212b58c4141a3b7be86feb91530 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d380c212b58c4141a3b7be86feb91530 2024-11-16T05:58:18,269 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into d380c212b58c4141a3b7be86feb91530(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:18,269 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736698232; duration=0sec 2024-11-16T05:58:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:18,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:18,270 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:18,270 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:18,270 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,270 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/6d10385408f549eca4e5b7b3775a71a9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.0 K 2024-11-16T05:58:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742343_1519 (size=12151) 2024-11-16T05:58:18,271 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d10385408f549eca4e5b7b3775a71a9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731736694815 2024-11-16T05:58:18,271 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/93ada95e5147446bb1688f8e76b5582f 2024-11-16T05:58:18,271 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24382b04b6e644618eb14ccb3c1736a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731736695533 2024-11-16T05:58:18,272 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41ce6d868b914f779e9d26f7e33c5df9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:18,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d74caec95c954a198ad9e205aca116a6 is 50, key is test_row_0/B:col10/1731736697444/Put/seqid=0 2024-11-16T05:58:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742344_1520 (size=12151) 2024-11-16T05:58:18,283 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:18,284 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/dd71f72506604e419f25833a05b6b793 is 50, key is test_row_0/C:col10/1731736696216/Put/seqid=0 2024-11-16T05:58:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742345_1521 (size=12663) 2024-11-16T05:58:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736758643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736758652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,667 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/0797e79802f8403d9425803238ce6545 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/0797e79802f8403d9425803238ce6545 2024-11-16T05:58:18,670 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 0797e79802f8403d9425803238ce6545(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:18,670 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:18,670 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736698232; duration=0sec 2024-11-16T05:58:18,670 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:18,670 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:18,682 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d74caec95c954a198ad9e205aca116a6 2024-11-16T05:58:18,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/215b203fc6874f4b80320e0ab8fc52b5 is 50, key is test_row_0/C:col10/1731736697444/Put/seqid=0 2024-11-16T05:58:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742346_1522 (size=12151) 2024-11-16T05:58:18,693 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/215b203fc6874f4b80320e0ab8fc52b5 2024-11-16T05:58:18,693 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/dd71f72506604e419f25833a05b6b793 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/dd71f72506604e419f25833a05b6b793 2024-11-16T05:58:18,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/93ada95e5147446bb1688f8e76b5582f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f 2024-11-16T05:58:18,697 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into dd71f72506604e419f25833a05b6b793(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:18,697 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:18,697 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736698233; duration=0sec 2024-11-16T05:58:18,697 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:18,697 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:18,699 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T05:58:18,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/d74caec95c954a198ad9e205aca116a6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6 2024-11-16T05:58:18,702 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T05:58:18,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/215b203fc6874f4b80320e0ab8fc52b5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5 2024-11-16T05:58:18,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T05:58:18,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c2babb3b1cd0035fe3947dd2fdeb4457 in 461ms, sequenceid=244, compaction requested=false 2024-11-16T05:58:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T05:58:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T05:58:18,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T05:58:18,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0740 sec 2024-11-16T05:58:18,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0770 sec 2024-11-16T05:58:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:18,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:18,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:18,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/2c860627bf0e40a08162311fe9ed9473 is 50, key is test_row_0/A:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T05:58:18,773 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742347_1523 (size=14691) 2024-11-16T05:58:18,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T05:58:18,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:18,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:18,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736758809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736758818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:18,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:18,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:18,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736758923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:18,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736758933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:18,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:19,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736759133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736759153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/2c860627bf0e40a08162311fe9ed9473 2024-11-16T05:58:19,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/035f75007bcf4358add85c38764a858a is 50, key is test_row_0/B:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:19,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742348_1524 (size=12251) 2024-11-16T05:58:19,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736759453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736759462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/035f75007bcf4358add85c38764a858a 2024-11-16T05:58:19,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/c42d0bd8efdf45758d64cb025a24f94e is 50, key is test_row_0/C:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:19,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742349_1525 (size=12251) 2024-11-16T05:58:19,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54494 deadline: 1731736759753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,763 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731736759773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,778 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in c2babb3b1cd0035fe3947dd2fdeb4457 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:19,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736759973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736759973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:19,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/c42d0bd8efdf45758d64cb025a24f94e 2024-11-16T05:58:20,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/2c860627bf0e40a08162311fe9ed9473 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473 2024-11-16T05:58:20,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:20,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473, entries=200, sequenceid=262, filesize=14.3 K 2024-11-16T05:58:20,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/035f75007bcf4358add85c38764a858a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a 2024-11-16T05:58:20,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T05:58:20,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/c42d0bd8efdf45758d64cb025a24f94e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e 2024-11-16T05:58:20,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T05:58:20,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1252ms, sequenceid=262, compaction requested=true 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:20,016 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:20,016 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:20,017 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,017 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,017 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/0797e79802f8403d9425803238ce6545, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.2 K 2024-11-16T05:58:20,017 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d380c212b58c4141a3b7be86feb91530, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=38.6 K 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0797e79802f8403d9425803238ce6545, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d380c212b58c4141a3b7be86feb91530, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ada95e5147446bb1688f8e76b5582f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736697406 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting d74caec95c954a198ad9e205aca116a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736697406 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c860627bf0e40a08162311fe9ed9473, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:20,017 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 035f75007bcf4358add85c38764a858a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:20,022 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:20,023 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/549a659a16654ba8bc312eb2be81e73e is 50, key is test_row_0/B:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:20,023 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:20,024 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/9b8ce4ec464f471b9fbdc14f04bc700d is 50, key is test_row_0/A:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:20,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742350_1526 (size=12865) 2024-11-16T05:58:20,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742351_1527 (size=12865) 2024-11-16T05:58:20,040 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/9b8ce4ec464f471b9fbdc14f04bc700d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9b8ce4ec464f471b9fbdc14f04bc700d 2024-11-16T05:58:20,041 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/549a659a16654ba8bc312eb2be81e73e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/549a659a16654ba8bc312eb2be81e73e 2024-11-16T05:58:20,048 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into 9b8ce4ec464f471b9fbdc14f04bc700d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:20,048 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:20,048 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736700016; duration=0sec 2024-11-16T05:58:20,048 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:20,048 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:20,048 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:20,049 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into 549a659a16654ba8bc312eb2be81e73e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:20,049 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:20,049 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736700016; duration=0sec 2024-11-16T05:58:20,049 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:20,049 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:20,049 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:20,049 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:20,050 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,050 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/dd71f72506604e419f25833a05b6b793, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.2 K 2024-11-16T05:58:20,050 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd71f72506604e419f25833a05b6b793, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731736696207 2024-11-16T05:58:20,050 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215b203fc6874f4b80320e0ab8fc52b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731736697406 2024-11-16T05:58:20,050 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42d0bd8efdf45758d64cb025a24f94e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:20,059 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:20,060 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/ea457ccd33374af0872c8f0781e155d6 is 50, key is test_row_0/C:col10/1731736698634/Put/seqid=0 2024-11-16T05:58:20,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742352_1528 (size=12865) 2024-11-16T05:58:20,078 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/ea457ccd33374af0872c8f0781e155d6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/ea457ccd33374af0872c8f0781e155d6 2024-11-16T05:58:20,082 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into ea457ccd33374af0872c8f0781e155d6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:20,082 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:20,082 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736700016; duration=0sec 2024-11-16T05:58:20,083 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:20,083 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:20,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T05:58:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:20,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d730de5ec7ec445182e263569f9a0511 is 50, key is test_row_0/A:col10/1731736698807/Put/seqid=0 2024-11-16T05:58:20,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742353_1529 (size=12301) 2024-11-16T05:58:20,593 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d730de5ec7ec445182e263569f9a0511 2024-11-16T05:58:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6f3da9d1ed934f7e821bb9689e9ef4d4 is 50, key is test_row_0/B:col10/1731736698807/Put/seqid=0 2024-11-16T05:58:20,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742354_1530 (size=12301) 2024-11-16T05:58:20,659 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6f3da9d1ed934f7e821bb9689e9ef4d4 2024-11-16T05:58:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 is 50, key is test_row_0/C:col10/1731736698807/Put/seqid=0 2024-11-16T05:58:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742355_1531 (size=12301) 2024-11-16T05:58:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:20,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736761059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736761061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 2024-11-16T05:58:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/d730de5ec7ec445182e263569f9a0511 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511 2024-11-16T05:58:21,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T05:58:21,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/6f3da9d1ed934f7e821bb9689e9ef4d4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4 2024-11-16T05:58:21,123 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T05:58:21,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 2024-11-16T05:58:21,128 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T05:58:21,129 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c2babb3b1cd0035fe3947dd2fdeb4457 in 973ms, sequenceid=282, compaction requested=false 2024-11-16T05:58:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T05:58:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T05:58:21,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T05:58:21,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3560 sec 2024-11-16T05:58:21,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3610 sec 2024-11-16T05:58:21,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:21,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/3b2e9c0ebd7b495687daecfa15143e9c is 50, key is test_row_0/A:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:21,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742356_1532 (size=14741) 2024-11-16T05:58:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736761299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736761299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736761424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736761424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/3b2e9c0ebd7b495687daecfa15143e9c 2024-11-16T05:58:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736761633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736761633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/5fc73c6b13d34b56b046ee88ffebc16d is 50, key is test_row_0/B:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:21,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742357_1533 (size=12301) 2024-11-16T05:58:21,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/5fc73c6b13d34b56b046ee88ffebc16d 2024-11-16T05:58:21,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/61a78a07d4ed4220a7c1f130da14fc8f is 50, key is test_row_0/C:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742358_1534 (size=12301) 2024-11-16T05:58:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736761953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736761953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/61a78a07d4ed4220a7c1f130da14fc8f 2024-11-16T05:58:22,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/3b2e9c0ebd7b495687daecfa15143e9c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c 2024-11-16T05:58:22,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c, entries=200, sequenceid=298, filesize=14.4 K 2024-11-16T05:58:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/5fc73c6b13d34b56b046ee88ffebc16d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d 2024-11-16T05:58:22,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-16T05:58:22,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/61a78a07d4ed4220a7c1f130da14fc8f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f 2024-11-16T05:58:22,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-16T05:58:22,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c2babb3b1cd0035fe3947dd2fdeb4457 in 948ms, sequenceid=298, compaction requested=true 2024-11-16T05:58:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:22,137 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:22,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:22,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:22,138 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:22,138 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:22,138 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9b8ce4ec464f471b9fbdc14f04bc700d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=39.0 K 2024-11-16T05:58:22,138 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/549a659a16654ba8bc312eb2be81e73e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.6 K 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 549a659a16654ba8bc312eb2be81e73e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:22,138 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8ce4ec464f471b9fbdc14f04bc700d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:22,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3da9d1ed934f7e821bb9689e9ef4d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731736698792 2024-11-16T05:58:22,139 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d730de5ec7ec445182e263569f9a0511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731736698792 2024-11-16T05:58:22,139 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc73c6b13d34b56b046ee88ffebc16d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:22,139 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2e9c0ebd7b495687daecfa15143e9c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:22,149 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:22,149 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:22,150 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a688486edf3944ea839d05d7795f7a8d is 50, key is test_row_0/B:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:22,150 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/ebe32f1a1d1145c8a751c124a6b72ab7 is 50, key is test_row_0/A:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742359_1535 (size=13017) 2024-11-16T05:58:22,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742360_1536 (size=13017) 2024-11-16T05:58:22,174 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a688486edf3944ea839d05d7795f7a8d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a688486edf3944ea839d05d7795f7a8d 2024-11-16T05:58:22,180 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into a688486edf3944ea839d05d7795f7a8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:22,180 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:22,180 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736702137; duration=0sec 2024-11-16T05:58:22,180 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:22,180 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:22,180 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:22,181 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:22,181 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:22,181 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:22,181 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/ea457ccd33374af0872c8f0781e155d6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.6 K 2024-11-16T05:58:22,181 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ea457ccd33374af0872c8f0781e155d6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731736698634 2024-11-16T05:58:22,181 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4a7e7dcc4349dc8d02f1ab5ec158d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731736698792 2024-11-16T05:58:22,182 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a78a07d4ed4220a7c1f130da14fc8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:22,189 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:22,189 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2452604d18db44a8b376da38c9d0a6fb is 50, key is test_row_0/C:col10/1731736701037/Put/seqid=0 2024-11-16T05:58:22,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742361_1537 (size=13017) 2024-11-16T05:58:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:22,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T05:58:22,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:22,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/75ac131e571d416d9d57c5cfb2f9b4fd is 50, key is test_row_0/A:col10/1731736701277/Put/seqid=0 2024-11-16T05:58:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742362_1538 (size=14741) 2024-11-16T05:58:22,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736762520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736762533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,558 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/ebe32f1a1d1145c8a751c124a6b72ab7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ebe32f1a1d1145c8a751c124a6b72ab7 2024-11-16T05:58:22,561 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into ebe32f1a1d1145c8a751c124a6b72ab7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:22,561 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:22,561 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736702137; duration=0sec 2024-11-16T05:58:22,561 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:22,561 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:22,599 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2452604d18db44a8b376da38c9d0a6fb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2452604d18db44a8b376da38c9d0a6fb 2024-11-16T05:58:22,605 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 2452604d18db44a8b376da38c9d0a6fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:22,605 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:22,605 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736702138; duration=0sec 2024-11-16T05:58:22,605 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:22,605 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736762643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736762643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736762853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736762853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:22,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/75ac131e571d416d9d57c5cfb2f9b4fd 2024-11-16T05:58:22,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/1bdab9df2dc84b98ab132b8629ed9cee is 50, key is test_row_0/B:col10/1731736701277/Put/seqid=0 2024-11-16T05:58:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742363_1539 (size=12301) 2024-11-16T05:58:22,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/1bdab9df2dc84b98ab132b8629ed9cee 2024-11-16T05:58:22,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/58ee5653e5d9451984962c7de7dd1d75 is 50, key is test_row_0/C:col10/1731736701277/Put/seqid=0 2024-11-16T05:58:22,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742364_1540 (size=12301) 2024-11-16T05:58:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T05:58:22,943 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:22,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T05:58:22,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T05:58:22,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:22,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T05:58:23,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T05:58:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736763163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736763172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T05:58:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T05:58:23,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/58ee5653e5d9451984962c7de7dd1d75 2024-11-16T05:58:23,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/75ac131e571d416d9d57c5cfb2f9b4fd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd 2024-11-16T05:58:23,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd, entries=200, sequenceid=317, filesize=14.4 K 2024-11-16T05:58:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/1bdab9df2dc84b98ab132b8629ed9cee as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee 2024-11-16T05:58:23,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee, entries=150, sequenceid=317, filesize=12.0 K 2024-11-16T05:58:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/58ee5653e5d9451984962c7de7dd1d75 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75 2024-11-16T05:58:23,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75, entries=150, sequenceid=317, filesize=12.0 K 2024-11-16T05:58:23,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c2babb3b1cd0035fe3947dd2fdeb4457 in 885ms, sequenceid=317, compaction requested=false 2024-11-16T05:58:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:23,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T05:58:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,407 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T05:58:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:23,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/6f4f0e6097994238b49af14ca7e36b44 is 50, key is test_row_0/A:col10/1731736702532/Put/seqid=0 2024-11-16T05:58:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742365_1541 (size=12301) 2024-11-16T05:58:23,435 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/6f4f0e6097994238b49af14ca7e36b44 2024-11-16T05:58:23,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9cb1f0911604433fbbcd6760b36efa57 is 50, key is test_row_0/B:col10/1731736702532/Put/seqid=0 2024-11-16T05:58:23,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742366_1542 (size=12301) 2024-11-16T05:58:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T05:58:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:23,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736763738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736763747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9cb1f0911604433fbbcd6760b36efa57 2024-11-16T05:58:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/1a1bae4eec984ce1837aa2ce59c4825d is 50, key is test_row_0/C:col10/1731736702532/Put/seqid=0 2024-11-16T05:58:23,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742367_1543 (size=12301) 2024-11-16T05:58:23,864 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/1a1bae4eec984ce1837aa2ce59c4825d 2024-11-16T05:58:23,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/6f4f0e6097994238b49af14ca7e36b44 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44 2024-11-16T05:58:23,871 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:58:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/9cb1f0911604433fbbcd6760b36efa57 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57 2024-11-16T05:58:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736763863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736763864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:23,875 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:58:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/1a1bae4eec984ce1837aa2ce59c4825d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d 2024-11-16T05:58:23,877 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T05:58:23,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c2babb3b1cd0035fe3947dd2fdeb4457 in 473ms, sequenceid=333, compaction requested=true 2024-11-16T05:58:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T05:58:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T05:58:23,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T05:58:23,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 935 msec 2024-11-16T05:58:23,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 939 msec 2024-11-16T05:58:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T05:58:24,083 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:24,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T05:58:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:24,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:24,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:24,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:24,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:24,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/481b755fdb7c4de4bbf8ec62bfe0827a is 50, key is test_row_0/A:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:24,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742368_1544 (size=17181) 2024-11-16T05:58:24,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736764131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736764140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:24,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736764252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736764252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736764463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736764463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/481b755fdb7c4de4bbf8ec62bfe0827a 2024-11-16T05:58:24,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/70a04fb14ffd4af6ad3ecd3f563c02cc is 50, key is test_row_0/B:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:24,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742369_1545 (size=12301) 2024-11-16T05:58:24,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/70a04fb14ffd4af6ad3ecd3f563c02cc 2024-11-16T05:58:24,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/e129cf77ac68475b90916671a337786b is 50, key is test_row_0/C:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742370_1546 (size=12301) 2024-11-16T05:58:24,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:24,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736764784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736764785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:24,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:24,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:24,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/e129cf77ac68475b90916671a337786b 2024-11-16T05:58:24,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/481b755fdb7c4de4bbf8ec62bfe0827a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a 2024-11-16T05:58:24,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a, entries=250, sequenceid=353, filesize=16.8 K 2024-11-16T05:58:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/70a04fb14ffd4af6ad3ecd3f563c02cc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc 2024-11-16T05:58:24,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:58:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/e129cf77ac68475b90916671a337786b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b 2024-11-16T05:58:25,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T05:58:25,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c2babb3b1cd0035fe3947dd2fdeb4457 in 911ms, sequenceid=353, compaction requested=true 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:25,003 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:25,003 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:25,004 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:25,004 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:25,004 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:25,004 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:25,004 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:25,004 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:25,004 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a688486edf3944ea839d05d7795f7a8d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=48.8 K 2024-11-16T05:58:25,004 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ebe32f1a1d1145c8a751c124a6b72ab7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=55.9 K 2024-11-16T05:58:25,005 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a688486edf3944ea839d05d7795f7a8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:25,005 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe32f1a1d1145c8a751c124a6b72ab7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:25,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:25,006 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bdab9df2dc84b98ab132b8629ed9cee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731736701256 2024-11-16T05:58:25,006 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T05:58:25,006 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75ac131e571d416d9d57c5cfb2f9b4fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731736701256 2024-11-16T05:58:25,006 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb1f0911604433fbbcd6760b36efa57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731736702495 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:25,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:25,006 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a04fb14ffd4af6ad3ecd3f563c02cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703736 2024-11-16T05:58:25,006 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4f0e6097994238b49af14ca7e36b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731736702495 2024-11-16T05:58:25,007 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 481b755fdb7c4de4bbf8ec62bfe0827a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703735 2024-11-16T05:58:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/574e7d37be414275a31863cbcc333783 is 50, key is test_row_0/A:col10/1731736704128/Put/seqid=0 2024-11-16T05:58:25,044 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#472 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:25,044 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/c6c70cdc116b4aa698985eedc89c867f is 50, key is test_row_0/A:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:25,052 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:25,052 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a0cedca305c94eed9c85aa4fe2aacb5d is 50, key is test_row_0/B:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:25,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742371_1547 (size=12301) 2024-11-16T05:58:25,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742373_1549 (size=13153) 2024-11-16T05:58:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742372_1548 (size=13153) 2024-11-16T05:58:25,075 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/c6c70cdc116b4aa698985eedc89c867f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/c6c70cdc116b4aa698985eedc89c867f 2024-11-16T05:58:25,080 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into c6c70cdc116b4aa698985eedc89c867f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:25,080 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,080 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=12, startTime=1731736705003; duration=0sec 2024-11-16T05:58:25,080 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:25,080 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:25,080 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:25,081 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:25,081 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:25,082 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:25,082 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2452604d18db44a8b376da38c9d0a6fb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=48.8 K 2024-11-16T05:58:25,082 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2452604d18db44a8b376da38c9d0a6fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731736701037 2024-11-16T05:58:25,082 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ee5653e5d9451984962c7de7dd1d75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731736701256 2024-11-16T05:58:25,082 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1bae4eec984ce1837aa2ce59c4825d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731736702495 2024-11-16T05:58:25,083 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e129cf77ac68475b90916671a337786b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703736 2024-11-16T05:58:25,090 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:25,091 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/8ad9984077334c93924516fcb6b282ea is 50, key is test_row_0/C:col10/1731736703736/Put/seqid=0 2024-11-16T05:58:25,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742374_1550 (size=13153) 2024-11-16T05:58:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:25,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. as already flushing 2024-11-16T05:58:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54506 deadline: 1731736765392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731736765404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 , the old value is region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., hostname=3456ee6a3164,45735,1731736558702, seqNum=2 because the exception is null or not the one we care about 2024-11-16T05:58:25,460 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/574e7d37be414275a31863cbcc333783 2024-11-16T05:58:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/627717df0b124943b4236f836bc217ad is 50, key is test_row_0/B:col10/1731736704128/Put/seqid=0 2024-11-16T05:58:25,472 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a0cedca305c94eed9c85aa4fe2aacb5d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a0cedca305c94eed9c85aa4fe2aacb5d 2024-11-16T05:58:25,476 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into a0cedca305c94eed9c85aa4fe2aacb5d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:25,476 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,477 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=12, startTime=1731736705003; duration=0sec 2024-11-16T05:58:25,477 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:25,477 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:25,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742375_1551 (size=12301) 2024-11-16T05:58:25,479 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/627717df0b124943b4236f836bc217ad 2024-11-16T05:58:25,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/37f2fcb097dd41678584780c03e9f7f2 is 50, key is test_row_0/C:col10/1731736704128/Put/seqid=0 2024-11-16T05:58:25,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742376_1552 (size=12301) 2024-11-16T05:58:25,495 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/37f2fcb097dd41678584780c03e9f7f2 2024-11-16T05:58:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/574e7d37be414275a31863cbcc333783 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783 2024-11-16T05:58:25,508 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783, entries=150, sequenceid=365, filesize=12.0 K 2024-11-16T05:58:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/627717df0b124943b4236f836bc217ad as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad 2024-11-16T05:58:25,513 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad, entries=150, sequenceid=365, filesize=12.0 K 2024-11-16T05:58:25,514 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/8ad9984077334c93924516fcb6b282ea as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/8ad9984077334c93924516fcb6b282ea 2024-11-16T05:58:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/37f2fcb097dd41678584780c03e9f7f2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2 2024-11-16T05:58:25,518 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 8ad9984077334c93924516fcb6b282ea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:25,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2, entries=150, sequenceid=365, filesize=12.0 K 2024-11-16T05:58:25,518 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,518 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=12, startTime=1731736705003; duration=0sec 2024-11-16T05:58:25,518 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:25,518 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:25,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c2babb3b1cd0035fe3947dd2fdeb4457 in 515ms, sequenceid=365, compaction requested=false 2024-11-16T05:58:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T05:58:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T05:58:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:25,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T05:58:25,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736705388, max timestamp=1731736705524Num uniques 859; 2024-11-16T05:58:25,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:25,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736705388, max timestamp=1731736705524Num uniques 859; 2024-11-16T05:58:25,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:25,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736705388, max timestamp=1731736705524Num uniques 859; 2024-11-16T05:58:25,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:25,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T05:58:25,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-16T05:58:25,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/09947cc15d1b494db07d0b9e284407a8 is 50, key is test_row_0/A:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:25,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4460 sec 2024-11-16T05:58:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742377_1553 (size=17181) 2024-11-16T05:58:25,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/09947cc15d1b494db07d0b9e284407a8 2024-11-16T05:58:25,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a2b9aaf4654f4f3e9240208c572d9226 is 50, key is test_row_0/B:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:25,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742378_1554 (size=12301) 2024-11-16T05:58:25,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a2b9aaf4654f4f3e9240208c572d9226 2024-11-16T05:58:25,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/898a1c786e72461f994dedb7f7ff441c is 50, key is test_row_0/C:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:25,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742379_1555 (size=12301) 2024-11-16T05:58:25,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736705587, max timestamp=1731736705600Num uniques 859; 2024-11-16T05:58:25,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736705587, max timestamp=1731736705600Num uniques 859; 2024-11-16T05:58:25,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:25,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:25,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736705587, max timestamp=1731736705600Num uniques 859; 2024-11-16T05:58:25,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:25,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731736705656, max timestamp=1731736705682Num uniques 1068; 2024-11-16T05:58:25,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T05:58:25,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731736705656, max timestamp=1731736705682Num uniques 1068; 2024-11-16T05:58:25,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T05:58:25,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731736705656, max timestamp=1731736705682Num uniques 1068; 2024-11-16T05:58:25,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T05:58:25,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736705730, max timestamp=1731736705778Num uniques 1077; 2024-11-16T05:58:25,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T05:58:25,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736705730, max timestamp=1731736705778Num uniques 1077; 2024-11-16T05:58:25,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736705730, max timestamp=1731736705778Num uniques 1077; 2024-11-16T05:58:25,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T05:58:25,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T05:58:25,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736705864, max timestamp=1731736705900Num uniques 1086; 2024-11-16T05:58:25,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736705864, max timestamp=1731736705900Num uniques 1086; 2024-11-16T05:58:25,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:25,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:25,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736705864, max timestamp=1731736705900Num uniques 1086; 2024-11-16T05:58:25,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:25,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/898a1c786e72461f994dedb7f7ff441c 2024-11-16T05:58:25,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/09947cc15d1b494db07d0b9e284407a8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8 2024-11-16T05:58:25,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8, entries=250, sequenceid=390, filesize=16.8 K 2024-11-16T05:58:25,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/a2b9aaf4654f4f3e9240208c572d9226 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226 2024-11-16T05:58:25,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226, entries=150, sequenceid=390, filesize=12.0 K 2024-11-16T05:58:25,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/898a1c786e72461f994dedb7f7ff441c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c 2024-11-16T05:58:25,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c, entries=150, sequenceid=390, filesize=12.0 K 2024-11-16T05:58:25,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=107.34 KB/109920 for c2babb3b1cd0035fe3947dd2fdeb4457 in 473ms, sequenceid=390, compaction requested=true 2024-11-16T05:58:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:25,999 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:26,000 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2babb3b1cd0035fe3947dd2fdeb4457:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/B is initiating minor compaction (all files) 2024-11-16T05:58:26,001 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/B in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:26,001 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a0cedca305c94eed9c85aa4fe2aacb5d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.9 K 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/A is initiating minor compaction (all files) 2024-11-16T05:58:26,001 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/A in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:26,001 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/c6c70cdc116b4aa698985eedc89c867f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=41.6 K 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a0cedca305c94eed9c85aa4fe2aacb5d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703736 2024-11-16T05:58:26,001 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c70cdc116b4aa698985eedc89c867f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703736 2024-11-16T05:58:26,002 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 627717df0b124943b4236f836bc217ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731736704128 2024-11-16T05:58:26,002 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 574e7d37be414275a31863cbcc333783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731736704128 2024-11-16T05:58:26,002 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b9aaf4654f4f3e9240208c572d9226, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731736705388 2024-11-16T05:58:26,002 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09947cc15d1b494db07d0b9e284407a8, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731736705388 2024-11-16T05:58:26,003 INFO [Thread-2117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2117. 2024-11-16T05:58:26,003 DEBUG [Thread-2117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,003 DEBUG [Thread-2117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,003 DEBUG [Thread-2117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:26,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=114.05 KB heapSize=322.69 KB 2024-11-16T05:58:26,006 INFO [Thread-2111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2111. 2024-11-16T05:58:26,006 DEBUG [Thread-2111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,006 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,006 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:26,008 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-11-16T05:58:26,008 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:26,008 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:26,008 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:26,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:26,011 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#A#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:26,012 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/4e3fc4095b2e4a6d9f59a99deee2816f is 50, key is test_row_0/A:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:26,017 INFO [Thread-2104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2104. 2024-11-16T05:58:26,017 DEBUG [Thread-2104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,017 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,017 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,018 INFO [Thread-2115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2115. 2024-11-16T05:58:26,018 DEBUG [Thread-2115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,018 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,018 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,020 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#B#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:26,020 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/ed78e750aacf44acaecec1d5ef937c60 is 50, key is test_row_0/B:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:26,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/4e9032669c9f4dcbbb175f72477362dc is 50, key is test_row_0/A:col10/1731736706000/Put/seqid=0 2024-11-16T05:58:26,024 INFO [Thread-2113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2113. 2024-11-16T05:58:26,024 DEBUG [Thread-2113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,024 DEBUG [Thread-2113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,024 DEBUG [Thread-2113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,029 INFO [Thread-2119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2119. 2024-11-16T05:58:26,029 DEBUG [Thread-2119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,029 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,029 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742381_1557 (size=13255) 2024-11-16T05:58:26,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742382_1558 (size=14741) 2024-11-16T05:58:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742380_1556 (size=13255) 2024-11-16T05:58:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T05:58:26,233 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:26,438 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/ed78e750aacf44acaecec1d5ef937c60 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/ed78e750aacf44acaecec1d5ef937c60 2024-11-16T05:58:26,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/4e9032669c9f4dcbbb175f72477362dc 2024-11-16T05:58:26,444 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/B of c2babb3b1cd0035fe3947dd2fdeb4457 into ed78e750aacf44acaecec1d5ef937c60(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:26,444 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:26,444 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/B, priority=13, startTime=1731736705999; duration=0sec 2024-11-16T05:58:26,444 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:26,444 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:B 2024-11-16T05:58:26,444 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:26,445 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:26,445 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): c2babb3b1cd0035fe3947dd2fdeb4457/C is initiating minor compaction (all files) 2024-11-16T05:58:26,445 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2babb3b1cd0035fe3947dd2fdeb4457/C in TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:26,445 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/8ad9984077334c93924516fcb6b282ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp, totalSize=36.9 K 2024-11-16T05:58:26,446 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad9984077334c93924516fcb6b282ea, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731736703736 2024-11-16T05:58:26,446 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f2fcb097dd41678584780c03e9f7f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731736704128 2024-11-16T05:58:26,446 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 898a1c786e72461f994dedb7f7ff441c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731736705388 2024-11-16T05:58:26,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/fa749e50b5eb44a687cff1c8b9cbe1d3 is 50, key is test_row_0/B:col10/1731736706000/Put/seqid=0 2024-11-16T05:58:26,451 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/4e3fc4095b2e4a6d9f59a99deee2816f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e3fc4095b2e4a6d9f59a99deee2816f 2024-11-16T05:58:26,453 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2babb3b1cd0035fe3947dd2fdeb4457#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:26,454 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2f4130bd596f46d18aef01cc3412cab0 is 50, key is test_row_0/C:col10/1731736705525/Put/seqid=0 2024-11-16T05:58:26,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742383_1559 (size=12301) 2024-11-16T05:58:26,455 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/A of c2babb3b1cd0035fe3947dd2fdeb4457 into 4e3fc4095b2e4a6d9f59a99deee2816f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:26,455 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:26,455 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/A, priority=13, startTime=1731736705999; duration=0sec 2024-11-16T05:58:26,456 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:26,456 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:A 2024-11-16T05:58:26,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/fa749e50b5eb44a687cff1c8b9cbe1d3 2024-11-16T05:58:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742384_1560 (size=13255) 2024-11-16T05:58:26,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/81960dd6cc6046a79b349cffac970f2a is 50, key is test_row_0/C:col10/1731736706000/Put/seqid=0 2024-11-16T05:58:26,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742385_1561 (size=12301) 2024-11-16T05:58:26,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/81960dd6cc6046a79b349cffac970f2a 2024-11-16T05:58:26,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/4e9032669c9f4dcbbb175f72477362dc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e9032669c9f4dcbbb175f72477362dc 2024-11-16T05:58:26,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e9032669c9f4dcbbb175f72477362dc, entries=200, sequenceid=475, filesize=14.4 K 2024-11-16T05:58:26,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/fa749e50b5eb44a687cff1c8b9cbe1d3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/fa749e50b5eb44a687cff1c8b9cbe1d3 2024-11-16T05:58:26,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/fa749e50b5eb44a687cff1c8b9cbe1d3, entries=150, sequenceid=475, filesize=12.0 K 2024-11-16T05:58:26,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/81960dd6cc6046a79b349cffac970f2a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/81960dd6cc6046a79b349cffac970f2a 2024-11-16T05:58:26,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/81960dd6cc6046a79b349cffac970f2a, entries=150, sequenceid=475, filesize=12.0 K 2024-11-16T05:58:26,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~322.64 KB/330384, currentSize=6.71 KB/6870 for c2babb3b1cd0035fe3947dd2fdeb4457 in 489ms, sequenceid=475, compaction requested=false 2024-11-16T05:58:26,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:26,755 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-11-16T05:58:26,755 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:26,755 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,755 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:26,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:26,873 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2f4130bd596f46d18aef01cc3412cab0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2f4130bd596f46d18aef01cc3412cab0 2024-11-16T05:58:26,877 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2babb3b1cd0035fe3947dd2fdeb4457/C of c2babb3b1cd0035fe3947dd2fdeb4457 into 2f4130bd596f46d18aef01cc3412cab0(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:26,877 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2babb3b1cd0035fe3947dd2fdeb4457: 2024-11-16T05:58:26,877 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., storeName=c2babb3b1cd0035fe3947dd2fdeb4457/C, priority=13, startTime=1731736706000; duration=0sec 2024-11-16T05:58:26,877 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:26,877 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2babb3b1cd0035fe3947dd2fdeb4457:C 2024-11-16T05:58:27,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:58:29,814 INFO [Thread-2106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2106. 2024-11-16T05:58:29,814 DEBUG [Thread-2106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:29,814 DEBUG [Thread-2106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,814 DEBUG [Thread-2106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:29,864 INFO [Thread-2108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2108. 2024-11-16T05:58:29,864 DEBUG [Thread-2108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:29,864 DEBUG [Thread-2108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,864 DEBUG [Thread-2108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:58:29,864 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3301 rows 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3306 rows 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1101 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3297 rows 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3309 rows 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-16T05:58:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3310 rows 2024-11-16T05:58:29,865 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:58:29,865 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:58:29,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:29,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:29,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:58:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:29,869 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736709869"}]},"ts":"1731736709869"} 2024-11-16T05:58:29,870 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:58:29,870 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:58:29,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:58:29,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, UNASSIGN}] 2024-11-16T05:58:29,873 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, UNASSIGN 2024-11-16T05:58:29,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=c2babb3b1cd0035fe3947dd2fdeb4457, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:29,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, UNASSIGN because future has completed 2024-11-16T05:58:29,875 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:58:29,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:30,028 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:30,028 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:58:30,028 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing c2babb3b1cd0035fe3947dd2fdeb4457, disabling compactions & flushes 2024-11-16T05:58:30,028 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:30,028 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. after waiting 0 ms 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:30,029 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing c2babb3b1cd0035fe3947dd2fdeb4457 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=A 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=B 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2babb3b1cd0035fe3947dd2fdeb4457, store=C 2024-11-16T05:58:30,029 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:30,040 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30755b9451d144239dcd537dd3ec05f7 is 50, key is test_row_1/A:col10/1731736709863/Put/seqid=0 2024-11-16T05:58:30,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742386_1562 (size=9857) 2024-11-16T05:58:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:30,445 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30755b9451d144239dcd537dd3ec05f7 2024-11-16T05:58:30,454 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/034663aef67644e699b6ccaf52aefc9b is 50, key is test_row_1/B:col10/1731736709863/Put/seqid=0 2024-11-16T05:58:30,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742387_1563 (size=9857) 2024-11-16T05:58:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:30,859 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/034663aef67644e699b6ccaf52aefc9b 2024-11-16T05:58:30,868 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 is 50, key is test_row_1/C:col10/1731736709863/Put/seqid=0 2024-11-16T05:58:30,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742388_1564 (size=9857) 2024-11-16T05:58:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:31,272 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 2024-11-16T05:58:31,275 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/A/30755b9451d144239dcd537dd3ec05f7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30755b9451d144239dcd537dd3ec05f7 2024-11-16T05:58:31,279 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30755b9451d144239dcd537dd3ec05f7, entries=100, sequenceid=485, filesize=9.6 K 2024-11-16T05:58:31,280 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/B/034663aef67644e699b6ccaf52aefc9b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/034663aef67644e699b6ccaf52aefc9b 2024-11-16T05:58:31,284 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/034663aef67644e699b6ccaf52aefc9b, entries=100, sequenceid=485, filesize=9.6 K 2024-11-16T05:58:31,285 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/.tmp/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 2024-11-16T05:58:31,288 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3, entries=100, sequenceid=485, filesize=9.6 K 2024-11-16T05:58:31,288 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1259ms, sequenceid=485, compaction requested=true 2024-11-16T05:58:31,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d9e81a12220d43ad9bfb2ae0db9499b2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/a9bf151d7a224fff87ee2decdeb5c869, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/50cc3425ed474a8dbe3b8b7fbf0ec98a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e04ad4c1483a4876a13bb2b534f889c8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d380c212b58c4141a3b7be86feb91530, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9b8ce4ec464f471b9fbdc14f04bc700d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ebe32f1a1d1145c8a751c124a6b72ab7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/c6c70cdc116b4aa698985eedc89c867f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8] to archive 2024-11-16T05:58:31,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:31,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/40100f98d670415b9cbe0f8a9cf378a0 2024-11-16T05:58:31,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/756cce77292d4620a554c98aa6c923ce 2024-11-16T05:58:31,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/282695aaa2ae4ce5834ed7f0e0487cfc 2024-11-16T05:58:31,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d9e81a12220d43ad9bfb2ae0db9499b2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d9e81a12220d43ad9bfb2ae0db9499b2 2024-11-16T05:58:31,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30fdaab070704bbc88a197ec95c78b60 2024-11-16T05:58:31,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e92d826d321649a68ee59920f1c5ca62 2024-11-16T05:58:31,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/a9bf151d7a224fff87ee2decdeb5c869 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/a9bf151d7a224fff87ee2decdeb5c869 2024-11-16T05:58:31,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9dd7ca1b87674c46a957aed20dca6073 2024-11-16T05:58:31,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/adcd0504f0a9445bb267b738ca01c10a 2024-11-16T05:58:31,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/50cc3425ed474a8dbe3b8b7fbf0ec98a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/50cc3425ed474a8dbe3b8b7fbf0ec98a 2024-11-16T05:58:31,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/716d26fab5e848fdaaa8b4c79f9cca87 2024-11-16T05:58:31,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ecb7a8b30cc84065bc002df98a4817e5 2024-11-16T05:58:31,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e04ad4c1483a4876a13bb2b534f889c8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/e04ad4c1483a4876a13bb2b534f889c8 2024-11-16T05:58:31,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/7ea00a5443454b069015bf6d0f4ab022 2024-11-16T05:58:31,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/8ad96f316fc14b8baeedc572f390fbb4 2024-11-16T05:58:31,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d380c212b58c4141a3b7be86feb91530 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d380c212b58c4141a3b7be86feb91530 2024-11-16T05:58:31,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/93ada95e5147446bb1688f8e76b5582f 2024-11-16T05:58:31,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/2c860627bf0e40a08162311fe9ed9473 2024-11-16T05:58:31,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9b8ce4ec464f471b9fbdc14f04bc700d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/9b8ce4ec464f471b9fbdc14f04bc700d 2024-11-16T05:58:31,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/d730de5ec7ec445182e263569f9a0511 2024-11-16T05:58:31,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/3b2e9c0ebd7b495687daecfa15143e9c 2024-11-16T05:58:31,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ebe32f1a1d1145c8a751c124a6b72ab7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/ebe32f1a1d1145c8a751c124a6b72ab7 2024-11-16T05:58:31,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/75ac131e571d416d9d57c5cfb2f9b4fd 2024-11-16T05:58:31,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/6f4f0e6097994238b49af14ca7e36b44 2024-11-16T05:58:31,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/481b755fdb7c4de4bbf8ec62bfe0827a 2024-11-16T05:58:31,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/c6c70cdc116b4aa698985eedc89c867f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/c6c70cdc116b4aa698985eedc89c867f 2024-11-16T05:58:31,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/574e7d37be414275a31863cbcc333783 2024-11-16T05:58:31,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/09947cc15d1b494db07d0b9e284407a8 2024-11-16T05:58:31,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9482ad85997647f2b6f4721bb8280f27, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6cd8d9e4af964eb3beebf48ea7904fd9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3af04221f98548e6bff0b84e4e58a314, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6d59d2a8224440d889f9e7914ae53e38, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/0797e79802f8403d9425803238ce6545, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/549a659a16654ba8bc312eb2be81e73e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a688486edf3944ea839d05d7795f7a8d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a0cedca305c94eed9c85aa4fe2aacb5d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226] to archive 2024-11-16T05:58:31,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:31,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/25119416db12443f8120eeacf3cdde7a 2024-11-16T05:58:31,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/66574114d01d4d7bb1093e3ed7b67ebf 2024-11-16T05:58:31,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9482ad85997647f2b6f4721bb8280f27 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9482ad85997647f2b6f4721bb8280f27 2024-11-16T05:58:31,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3a67b70044b4400ab1f9a08a93f56f7d 2024-11-16T05:58:31,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/cbaa7ad01d9d429b952d5b3d582d04da 2024-11-16T05:58:31,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6cd8d9e4af964eb3beebf48ea7904fd9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6cd8d9e4af964eb3beebf48ea7904fd9 2024-11-16T05:58:31,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9707a3fbf941429d87bdd0cffdf5f703 2024-11-16T05:58:31,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/de21b3aa41cb411a9388cf037cc07286 2024-11-16T05:58:31,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3af04221f98548e6bff0b84e4e58a314 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/3af04221f98548e6bff0b84e4e58a314 2024-11-16T05:58:31,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/4d2cd2af3ff34021a7503920efce4bc2 2024-11-16T05:58:31,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/345db1287701464b8a958f722895ebdf 2024-11-16T05:58:31,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6d59d2a8224440d889f9e7914ae53e38 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6d59d2a8224440d889f9e7914ae53e38 2024-11-16T05:58:31,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d6684e7daf4a4a249af62e950e5e5afb 2024-11-16T05:58:31,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/60b36e602fb04cf1b9e75ff836acf44f 2024-11-16T05:58:31,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/0797e79802f8403d9425803238ce6545 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/0797e79802f8403d9425803238ce6545 2024-11-16T05:58:31,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/65d777b6d3b44228a42ca8c93878db0a 2024-11-16T05:58:31,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/d74caec95c954a198ad9e205aca116a6 2024-11-16T05:58:31,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/549a659a16654ba8bc312eb2be81e73e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/549a659a16654ba8bc312eb2be81e73e 2024-11-16T05:58:31,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/035f75007bcf4358add85c38764a858a 2024-11-16T05:58:31,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/6f3da9d1ed934f7e821bb9689e9ef4d4 2024-11-16T05:58:31,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a688486edf3944ea839d05d7795f7a8d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a688486edf3944ea839d05d7795f7a8d 2024-11-16T05:58:31,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/5fc73c6b13d34b56b046ee88ffebc16d 2024-11-16T05:58:31,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/1bdab9df2dc84b98ab132b8629ed9cee 2024-11-16T05:58:31,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/9cb1f0911604433fbbcd6760b36efa57 2024-11-16T05:58:31,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a0cedca305c94eed9c85aa4fe2aacb5d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a0cedca305c94eed9c85aa4fe2aacb5d 2024-11-16T05:58:31,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/70a04fb14ffd4af6ad3ecd3f563c02cc 2024-11-16T05:58:31,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/627717df0b124943b4236f836bc217ad 2024-11-16T05:58:31,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/a2b9aaf4654f4f3e9240208c572d9226 2024-11-16T05:58:31,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fe9fdbbe09fa41aea2f1f3f211eacab5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/262660dbee8b48c08ed5fb44317466e6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2eea3345def24f6aae0de5d46720cc50, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/6d10385408f549eca4e5b7b3775a71a9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/dd71f72506604e419f25833a05b6b793, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/ea457ccd33374af0872c8f0781e155d6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2452604d18db44a8b376da38c9d0a6fb, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/8ad9984077334c93924516fcb6b282ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c] to archive 2024-11-16T05:58:31,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:58:31,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/f5e1595770b54ba3b31f72c3d32eba28 2024-11-16T05:58:31,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/677fce07180f478796bc42f0d4c16aa9 2024-11-16T05:58:31,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fe9fdbbe09fa41aea2f1f3f211eacab5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fe9fdbbe09fa41aea2f1f3f211eacab5 2024-11-16T05:58:31,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b282430f10e842c4bfb5a9b3003236fa 2024-11-16T05:58:31,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/b4d5395b35364c8783f8a39207eef489 2024-11-16T05:58:31,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/262660dbee8b48c08ed5fb44317466e6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/262660dbee8b48c08ed5fb44317466e6 2024-11-16T05:58:31,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/3ae88e4a23014d15a879b613fe2ed5d8 2024-11-16T05:58:31,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d6f7d22e508f4272bd6c754688f23e42 2024-11-16T05:58:31,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2eea3345def24f6aae0de5d46720cc50 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2eea3345def24f6aae0de5d46720cc50 2024-11-16T05:58:31,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/998ea5f26109471a833b072875d4d28c 2024-11-16T05:58:31,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/d314ce585bf84c3abe2bfcc798705d70 2024-11-16T05:58:31,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/6d10385408f549eca4e5b7b3775a71a9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/6d10385408f549eca4e5b7b3775a71a9 2024-11-16T05:58:31,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/68f8f77699da4e1591e78ca887734944 2024-11-16T05:58:31,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/24382b04b6e644618eb14ccb3c1736a2 2024-11-16T05:58:31,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/dd71f72506604e419f25833a05b6b793 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/dd71f72506604e419f25833a05b6b793 2024-11-16T05:58:31,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/41ce6d868b914f779e9d26f7e33c5df9 2024-11-16T05:58:31,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/215b203fc6874f4b80320e0ab8fc52b5 2024-11-16T05:58:31,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/ea457ccd33374af0872c8f0781e155d6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/ea457ccd33374af0872c8f0781e155d6 2024-11-16T05:58:31,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/c42d0bd8efdf45758d64cb025a24f94e 2024-11-16T05:58:31,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/fc4a7e7dcc4349dc8d02f1ab5ec158d9 2024-11-16T05:58:31,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2452604d18db44a8b376da38c9d0a6fb to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2452604d18db44a8b376da38c9d0a6fb 2024-11-16T05:58:31,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/61a78a07d4ed4220a7c1f130da14fc8f 2024-11-16T05:58:31,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/58ee5653e5d9451984962c7de7dd1d75 2024-11-16T05:58:31,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/1a1bae4eec984ce1837aa2ce59c4825d 2024-11-16T05:58:31,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/8ad9984077334c93924516fcb6b282ea to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/8ad9984077334c93924516fcb6b282ea 2024-11-16T05:58:31,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/e129cf77ac68475b90916671a337786b 2024-11-16T05:58:31,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/37f2fcb097dd41678584780c03e9f7f2 2024-11-16T05:58:31,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/898a1c786e72461f994dedb7f7ff441c 2024-11-16T05:58:31,381 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/recovered.edits/488.seqid, newMaxSeqId=488, maxSeqId=1 2024-11-16T05:58:31,382 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. 2024-11-16T05:58:31,382 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for c2babb3b1cd0035fe3947dd2fdeb4457: Waiting for close lock at 1731736710028Running coprocessor pre-close hooks at 1731736710028Disabling compacts and flushes for region at 1731736710028Disabling writes for close at 1731736710029 (+1 ms)Obtaining lock to block concurrent updates at 1731736710029Preparing flush snapshotting stores in c2babb3b1cd0035fe3947dd2fdeb4457 at 1731736710029Finished memstore snapshotting TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731736710029Flushing stores of TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457. at 1731736710030 (+1 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/A: creating writer at 1731736710030Flushing c2babb3b1cd0035fe3947dd2fdeb4457/A: appending metadata at 1731736710039 (+9 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/A: closing flushed file at 1731736710039Flushing c2babb3b1cd0035fe3947dd2fdeb4457/B: creating writer at 1731736710451 (+412 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/B: appending metadata at 1731736710453 (+2 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/B: closing flushed file at 1731736710453Flushing c2babb3b1cd0035fe3947dd2fdeb4457/C: creating writer at 1731736710865 (+412 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/C: appending metadata at 1731736710868 (+3 ms)Flushing c2babb3b1cd0035fe3947dd2fdeb4457/C: closing flushed file at 1731736710868Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ca1c347: reopening flushed file at 1731736711275 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75327ec6: reopening flushed file at 1731736711279 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70ca91ff: reopening flushed file at 1731736711284 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c2babb3b1cd0035fe3947dd2fdeb4457 in 1259ms, sequenceid=485, compaction requested=true at 1731736711288 (+4 ms)Writing region close event to WAL at 1731736711379 (+91 ms)Running coprocessor post-close hooks at 1731736711382 (+3 ms)Closed at 1731736711382 2024-11-16T05:58:31,384 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:31,385 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=c2babb3b1cd0035fe3947dd2fdeb4457, regionState=CLOSED 2024-11-16T05:58:31,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:31,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T05:58:31,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure c2babb3b1cd0035fe3947dd2fdeb4457, server=3456ee6a3164,45735,1731736558702 in 1.5130 sec 2024-11-16T05:58:31,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-16T05:58:31,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2babb3b1cd0035fe3947dd2fdeb4457, UNASSIGN in 1.5170 sec 2024-11-16T05:58:31,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T05:58:31,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-16T05:58:31,395 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736711395"}]},"ts":"1731736711395"} 2024-11-16T05:58:31,396 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:58:31,397 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:58:31,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5310 sec 2024-11-16T05:58:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T05:58:32,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:32,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:58:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:58:32,020 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,023 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:32,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/recovered.edits] 2024-11-16T05:58:32,030 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30755b9451d144239dcd537dd3ec05f7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/30755b9451d144239dcd537dd3ec05f7 2024-11-16T05:58:32,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e3fc4095b2e4a6d9f59a99deee2816f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e3fc4095b2e4a6d9f59a99deee2816f 2024-11-16T05:58:32,035 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e9032669c9f4dcbbb175f72477362dc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/A/4e9032669c9f4dcbbb175f72477362dc 2024-11-16T05:58:32,038 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/034663aef67644e699b6ccaf52aefc9b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/034663aef67644e699b6ccaf52aefc9b 2024-11-16T05:58:32,040 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/ed78e750aacf44acaecec1d5ef937c60 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/ed78e750aacf44acaecec1d5ef937c60 2024-11-16T05:58:32,043 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/fa749e50b5eb44a687cff1c8b9cbe1d3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/B/fa749e50b5eb44a687cff1c8b9cbe1d3 2024-11-16T05:58:32,046 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2dcf3d7c20ee4bebb8bfbe46b02c07b3 2024-11-16T05:58:32,048 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2f4130bd596f46d18aef01cc3412cab0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/2f4130bd596f46d18aef01cc3412cab0 2024-11-16T05:58:32,050 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/81960dd6cc6046a79b349cffac970f2a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/C/81960dd6cc6046a79b349cffac970f2a 2024-11-16T05:58:32,054 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/recovered.edits/488.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457/recovered.edits/488.seqid 2024-11-16T05:58:32,054 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/c2babb3b1cd0035fe3947dd2fdeb4457 2024-11-16T05:58:32,055 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:58:32,058 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,061 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:58:32,064 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:58:32,066 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,066 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:58:32,066 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736712066"}]},"ts":"9223372036854775807"} 2024-11-16T05:58:32,069 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:58:32,069 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c2babb3b1cd0035fe3947dd2fdeb4457, NAME => 'TestAcidGuarantees,,1731736683780.c2babb3b1cd0035fe3947dd2fdeb4457.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:58:32,069 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:58:32,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736712069"}]},"ts":"9223372036854775807"} 2024-11-16T05:58:32,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:58:32,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-16T05:58:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T05:58:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:58:32,124 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:32,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=509 (was 539), ProcessCount=11 (was 11), AvailableMemoryMB=3390 (was 3388) - AvailableMemoryMB LEAK? - 2024-11-16T05:58:32,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=509, ProcessCount=11, AvailableMemoryMB=3389 2024-11-16T05:58:32,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:58:32,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:58:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:32,145 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:58:32,145 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:32,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-16T05:58:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:32,146 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:58:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742389_1565 (size=1061) 2024-11-16T05:58:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:32,555 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79 2024-11-16T05:58:32,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742390_1566 (size=53) 2024-11-16T05:58:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:32,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:58:32,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a01be5863776074fa8e10401262fcc00, disabling compactions & flushes 2024-11-16T05:58:32,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:32,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:32,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. after waiting 0 ms 2024-11-16T05:58:32,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:32,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:32,965 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a01be5863776074fa8e10401262fcc00: Waiting for close lock at 1731736712964Disabling compacts and flushes for region at 1731736712964Disabling writes for close at 1731736712964Writing region close event to WAL at 1731736712964Closed at 1731736712964 2024-11-16T05:58:32,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:58:32,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731736712965"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736712965"}]},"ts":"1731736712965"} 2024-11-16T05:58:32,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:58:32,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:58:32,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736712968"}]},"ts":"1731736712968"} 2024-11-16T05:58:32,969 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T05:58:32,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, ASSIGN}] 2024-11-16T05:58:32,970 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, ASSIGN 2024-11-16T05:58:32,971 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, ASSIGN; state=OFFLINE, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=false 2024-11-16T05:58:33,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:33,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, ASSIGN because future has completed 2024-11-16T05:58:33,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:33,277 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:33,277 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:58:33,278 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,278 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:58:33,278 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,278 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,279 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,280 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:33,280 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName A 2024-11-16T05:58:33,280 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:33,281 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:33,281 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,282 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:33,282 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName B 2024-11-16T05:58:33,282 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:33,283 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:33,283 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,284 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:33,284 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName C 2024-11-16T05:58:33,284 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:33,285 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:33,285 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:33,285 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,285 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,286 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,287 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,287 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,287 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:58:33,288 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,290 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:58:33,290 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened a01be5863776074fa8e10401262fcc00; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67559427, jitterRate=0.0067139118909835815}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:58:33,290 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:33,291 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for a01be5863776074fa8e10401262fcc00: Running coprocessor pre-open hook at 1731736713278Writing region info on filesystem at 1731736713278Initializing all the Stores at 1731736713278Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736713278Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736713279 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736713279Cleaning up temporary data from old regions at 1731736713287 (+8 ms)Running coprocessor post-open hooks at 1731736713290 (+3 ms)Region opened successfully at 1731736713291 (+1 ms) 2024-11-16T05:58:33,291 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., pid=136, masterSystemTime=1731736713274 2024-11-16T05:58:33,293 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:33,293 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:33,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:33,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:33,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T05:58:33,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 in 172 msec 2024-11-16T05:58:33,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-16T05:58:33,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, ASSIGN in 328 msec 2024-11-16T05:58:33,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:58:33,300 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736713300"}]},"ts":"1731736713300"} 2024-11-16T05:58:33,301 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T05:58:33,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:58:33,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-16T05:58:34,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-16T05:58:34,294 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:34,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:34,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:34,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:34,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:34,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:34,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3b5256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:34,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0571ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:34,302 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:34,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:34,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:58:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40477,1731736558105 2024-11-16T05:58:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23d1d9b7 2024-11-16T05:58:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:58:34,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:58:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T05:58:34,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:58:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T05:58:34,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742391_1567 (size=1097) 2024-11-16T05:58:34,721 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T05:58:34,721 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T05:58:34,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:58:34,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE}] 2024-11-16T05:58:34,725 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE 2024-11-16T05:58:34,726 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:34,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE because future has completed 2024-11-16T05:58:34,728 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:58:34,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:34,880 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:34,880 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:58:34,881 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing a01be5863776074fa8e10401262fcc00, disabling compactions & flushes 2024-11-16T05:58:34,881 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:34,881 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:34,881 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. after waiting 0 ms 2024-11-16T05:58:34,881 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:34,885 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:58:34,886 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:34,886 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for a01be5863776074fa8e10401262fcc00: Waiting for close lock at 1731736714881Running coprocessor pre-close hooks at 1731736714881Disabling compacts and flushes for region at 1731736714881Disabling writes for close at 1731736714881Writing region close event to WAL at 1731736714881Running coprocessor post-close hooks at 1731736714886 (+5 ms)Closed at 1731736714886 2024-11-16T05:58:34,886 WARN [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: a01be5863776074fa8e10401262fcc00 to self. 2024-11-16T05:58:34,888 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:34,889 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=CLOSED 2024-11-16T05:58:34,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:34,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T05:58:34,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 in 164 msec 2024-11-16T05:58:34,896 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,45735,1731736558702; forceNewPlan=false, retain=true 2024-11-16T05:58:35,047 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=OPENING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE because future has completed 2024-11-16T05:58:35,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:58:35,204 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,204 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:58:35,204 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,204 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:58:35,204 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,204 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,206 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,207 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:35,207 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName A 2024-11-16T05:58:35,208 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:35,208 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:35,209 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,209 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:35,209 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName B 2024-11-16T05:58:35,209 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:35,210 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:35,210 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,210 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T05:58:35,210 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a01be5863776074fa8e10401262fcc00 columnFamilyName C 2024-11-16T05:58:35,211 DEBUG [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:35,211 INFO [StoreOpener-a01be5863776074fa8e10401262fcc00-1 {}] regionserver.HStore(327): Store=a01be5863776074fa8e10401262fcc00/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:58:35,211 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,211 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,211 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,212 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,213 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,213 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,214 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T05:58:35,215 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,215 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened a01be5863776074fa8e10401262fcc00; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71457695, jitterRate=0.0648026317358017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T05:58:35,216 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,216 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for a01be5863776074fa8e10401262fcc00: Running coprocessor pre-open hook at 1731736715205Writing region info on filesystem at 1731736715205Initializing all the Stores at 1731736715205Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736715205Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736715206 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736715206Cleaning up temporary data from old regions at 1731736715213 (+7 ms)Running coprocessor post-open hooks at 1731736715216 (+3 ms)Region opened successfully at 1731736715216 2024-11-16T05:58:35,217 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., pid=141, masterSystemTime=1731736715201 2024-11-16T05:58:35,219 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,219 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,220 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40477 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,45735,1731736558702, table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:58:35,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:58:35,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-16T05:58:35,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 in 173 msec 2024-11-16T05:58:35,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-16T05:58:35,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, REOPEN/MOVE in 499 msec 2024-11-16T05:58:35,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T05:58:35,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-16T05:58:35,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-11-16T05:58:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T05:58:35,232 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:35,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7a7eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3196230f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,241 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4084fe10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728e177f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35cf224b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f11c4fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230b339b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,256 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd5fb9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,258 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27002a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebe97e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fd5107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,264 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7127fae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,266 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cf05f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf9dde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3336e3a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,270 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e9edbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,271 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3417795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1d7832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40477,-1 for getting cluster id 2024-11-16T05:58:35,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:58:35,274 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56d35825-0086-4de4-ab79-2eff0746a20d' 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56d35825-0086-4de4-ab79-2eff0746a20d" 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20aea82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40477,-1] 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:35,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:58:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462ad14f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:58:35,276 DEBUG [Thread-2529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,276 DEBUG [Thread-2512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,276 DEBUG [Thread-2527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,277 DEBUG [Thread-2516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,277 DEBUG [Thread-2520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,277 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,277 DEBUG [Thread-2518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,277 DEBUG [Thread-2523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:35,278 DEBUG [Thread-2525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,280 DEBUG [Thread-2514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:58:35,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,45735,1731736558702, seqNum=-1] 2024-11-16T05:58:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:58:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:58:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5] 2024-11-16T05:58:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-16T05:58:35,286 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:35,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:35,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:35,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c7c71b45aa3f469783bdc06fdef63aed_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736715287/Put/seqid=0 2024-11-16T05:58:35,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736715294, max timestamp=1731736715308Num uniques 859; 2024-11-16T05:58:35,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736715294, max timestamp=1731736715308Num uniques 859; 2024-11-16T05:58:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736715294, max timestamp=1731736715308Num uniques 859; 2024-11-16T05:58:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:35,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742392_1568 (size=14594) 2024-11-16T05:58:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736775322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736775323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736775324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736775325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736775326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736775433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736775433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736775433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736775433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736775443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:35,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736775643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736775643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736775643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736775643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736775653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:35,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c7c71b45aa3f469783bdc06fdef63aed_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7c71b45aa3f469783bdc06fdef63aed_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:35,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4fa19b5983584b99b5e9d624c590d9ad, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:35,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4fa19b5983584b99b5e9d624c590d9ad is 175, key is test_row_0/A:col10/1731736715287/Put/seqid=0 2024-11-16T05:58:35,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742393_1569 (size=39549) 2024-11-16T05:58:35,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:35,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:35,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736775963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4fa19b5983584b99b5e9d624c590d9ad 2024-11-16T05:58:36,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/af9c0319c92f48638321b4bc05a89451 is 50, key is test_row_0/B:col10/1731736715287/Put/seqid=0 2024-11-16T05:58:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742394_1570 (size=12001) 2024-11-16T05:58:36,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736776462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736776463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736776463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736776463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736776473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:36,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:36,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/af9c0319c92f48638321b4bc05a89451 2024-11-16T05:58:36,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/34703b03f03d464a8e9f6be8bcd4be9b is 50, key is test_row_0/C:col10/1731736715287/Put/seqid=0 2024-11-16T05:58:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742395_1571 (size=12001) 2024-11-16T05:58:36,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/34703b03f03d464a8e9f6be8bcd4be9b 2024-11-16T05:58:36,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4fa19b5983584b99b5e9d624c590d9ad as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad 2024-11-16T05:58:36,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad, entries=200, sequenceid=14, filesize=38.6 K 2024-11-16T05:58:36,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/af9c0319c92f48638321b4bc05a89451 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451 2024-11-16T05:58:36,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:58:36,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/34703b03f03d464a8e9f6be8bcd4be9b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b 2024-11-16T05:58:36,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T05:58:36,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a01be5863776074fa8e10401262fcc00 in 1300ms, sequenceid=14, compaction requested=false 2024-11-16T05:58:36,588 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T05:58:36,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:36,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-16T05:58:36,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:36,660 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:36,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697c8a4ccd7e748c1aa2c2d3052e8c338_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736715324/Put/seqid=0 2024-11-16T05:58:36,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742396_1572 (size=12154) 2024-11-16T05:58:36,898 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:58:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:37,082 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697c8a4ccd7e748c1aa2c2d3052e8c338_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697c8a4ccd7e748c1aa2c2d3052e8c338_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/6b259b868a7c43e6959beefd43ee48ae, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/6b259b868a7c43e6959beefd43ee48ae is 175, key is test_row_0/A:col10/1731736715324/Put/seqid=0 2024-11-16T05:58:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742397_1573 (size=30955) 2024-11-16T05:58:37,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/6b259b868a7c43e6959beefd43ee48ae 2024-11-16T05:58:37,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/6bb33f6de40c49689d0b7645b31e5ed5 is 50, key is test_row_0/B:col10/1731736715324/Put/seqid=0 2024-11-16T05:58:37,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742398_1574 (size=12001) 2024-11-16T05:58:37,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/6bb33f6de40c49689d0b7645b31e5ed5 2024-11-16T05:58:37,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/de58600ce0ab4f3282d7e41258f8b8ea is 50, key is test_row_0/C:col10/1731736715324/Put/seqid=0 2024-11-16T05:58:37,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742399_1575 (size=12001) 2024-11-16T05:58:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:37,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:37,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736777481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736777482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,488 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:37,488 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736777486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,516 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/de58600ce0ab4f3282d7e41258f8b8ea 2024-11-16T05:58:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/6b259b868a7c43e6959beefd43ee48ae as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae 2024-11-16T05:58:37,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae, entries=150, sequenceid=50, filesize=30.2 K 2024-11-16T05:58:37,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/6bb33f6de40c49689d0b7645b31e5ed5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5 2024-11-16T05:58:37,526 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T05:58:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/de58600ce0ab4f3282d7e41258f8b8ea as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea 2024-11-16T05:58:37,530 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T05:58:37,531 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a01be5863776074fa8e10401262fcc00 in 871ms, sequenceid=50, compaction requested=false 2024-11-16T05:58:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-16T05:58:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-16T05:58:37,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T05:58:37,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-11-16T05:58:37,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.2560 sec 2024-11-16T05:58:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:37,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:37,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116092c0e624ddb40b7b107d2d04e62a2f0_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742400_1576 (size=12154) 2024-11-16T05:58:37,610 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116092c0e624ddb40b7b107d2d04e62a2f0_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116092c0e624ddb40b7b107d2d04e62a2f0_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:37,613 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b5f79ffc67254edc8d2ca14ddb0d44cc, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:37,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b5f79ffc67254edc8d2ca14ddb0d44cc is 175, key is test_row_0/A:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:37,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742401_1577 (size=30955) 2024-11-16T05:58:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736777624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736777625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736717618, max timestamp=1731736717628Num uniques 859; 2024-11-16T05:58:37,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736717618, max timestamp=1731736717628Num uniques 859; 2024-11-16T05:58:37,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736717618, max timestamp=1731736717628Num uniques 859; 2024-11-16T05:58:37,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:37,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736777714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736777733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736777733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736777823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736777943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736777943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b5f79ffc67254edc8d2ca14ddb0d44cc 2024-11-16T05:58:38,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/35862b8b9d2d4638991077c736962293 is 50, key is test_row_0/B:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:38,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742402_1578 (size=12001) 2024-11-16T05:58:38,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736778033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736778253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736778253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736778342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/35862b8b9d2d4638991077c736962293 2024-11-16T05:58:38,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/b91f8a49c3b3441e8040edd54e908aa8 is 50, key is test_row_0/C:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:38,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742403_1579 (size=12001) 2024-11-16T05:58:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736778763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736778763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:38,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/b91f8a49c3b3441e8040edd54e908aa8 2024-11-16T05:58:38,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b5f79ffc67254edc8d2ca14ddb0d44cc as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc 2024-11-16T05:58:38,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc, entries=150, sequenceid=61, filesize=30.2 K 2024-11-16T05:58:38,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/35862b8b9d2d4638991077c736962293 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293 2024-11-16T05:58:38,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T05:58:38,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/b91f8a49c3b3441e8040edd54e908aa8 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8 2024-11-16T05:58:38,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T05:58:38,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a01be5863776074fa8e10401262fcc00 in 1260ms, sequenceid=61, compaction requested=true 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:38,855 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:38,855 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:38,856 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:38,856 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:38,856 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=35.2 K 2024-11-16T05:58:38,856 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=99.1 K 2024-11-16T05:58:38,856 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc] 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting af9c0319c92f48638321b4bc05a89451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736715284 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa19b5983584b99b5e9d624c590d9ad, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736715284 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb33f6de40c49689d0b7645b31e5ed5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736715321 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b259b868a7c43e6959beefd43ee48ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736715321 2024-11-16T05:58:38,856 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 35862b8b9d2d4638991077c736962293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:38,857 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f79ffc67254edc8d2ca14ddb0d44cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:38,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:38,861 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:38,869 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:38,869 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/604acee2c13f41b4bfff94a67e9ff240 is 50, key is test_row_0/B:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:38,878 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116bc161fb35ffd4934a91846518568ee01_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:38,881 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116bc161fb35ffd4934a91846518568ee01_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:38,881 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bc161fb35ffd4934a91846518568ee01_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:38,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b0b08a248e674022a6b4abeebbdf5676_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736717713/Put/seqid=0 2024-11-16T05:58:38,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742404_1580 (size=12104) 2024-11-16T05:58:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742406_1582 (size=14594) 2024-11-16T05:58:38,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:38,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742405_1581 (size=4469) 2024-11-16T05:58:38,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b0b08a248e674022a6b4abeebbdf5676_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b0b08a248e674022a6b4abeebbdf5676_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:38,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8a1693b42e3b43998ee96ff71262962d, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:38,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8a1693b42e3b43998ee96ff71262962d is 175, key is test_row_0/A:col10/1731736717713/Put/seqid=0 2024-11-16T05:58:38,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742407_1583 (size=39549) 2024-11-16T05:58:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736778901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736779013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736779222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,292 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:39,292 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/604acee2c13f41b4bfff94a67e9ff240 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/604acee2c13f41b4bfff94a67e9ff240 2024-11-16T05:58:39,292 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/d663168f9e7c4383ae44cb58da6b7511 is 175, key is test_row_0/A:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:39,297 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into 604acee2c13f41b4bfff94a67e9ff240(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:39,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8a1693b42e3b43998ee96ff71262962d 2024-11-16T05:58:39,297 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:39,297 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=13, startTime=1731736718855; duration=0sec 2024-11-16T05:58:39,297 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:39,297 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:39,297 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:39,299 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:39,299 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:39,299 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,299 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=35.2 K 2024-11-16T05:58:39,299 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 34703b03f03d464a8e9f6be8bcd4be9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731736715284 2024-11-16T05:58:39,300 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting de58600ce0ab4f3282d7e41258f8b8ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731736715321 2024-11-16T05:58:39,300 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b91f8a49c3b3441e8040edd54e908aa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:39,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/8d1834e9e99646938153d1231ace8a87 is 50, key is test_row_0/B:col10/1731736717713/Put/seqid=0 2024-11-16T05:58:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742409_1585 (size=12001) 2024-11-16T05:58:39,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/8d1834e9e99646938153d1231ace8a87 2024-11-16T05:58:39,317 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:39,318 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f32c19e28efc4eedb53bab0264c0a7a6 is 50, key is test_row_0/C:col10/1731736717485/Put/seqid=0 2024-11-16T05:58:39,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742408_1584 (size=31058) 2024-11-16T05:58:39,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742410_1586 (size=12104) 2024-11-16T05:58:39,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ddab84ca8c9a4894851a63dbada46e6d is 50, key is test_row_0/C:col10/1731736717713/Put/seqid=0 2024-11-16T05:58:39,324 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/d663168f9e7c4383ae44cb58da6b7511 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511 2024-11-16T05:58:39,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742411_1587 (size=12001) 2024-11-16T05:58:39,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ddab84ca8c9a4894851a63dbada46e6d 2024-11-16T05:58:39,329 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into d663168f9e7c4383ae44cb58da6b7511(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:39,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:39,329 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=13, startTime=1731736718855; duration=0sec 2024-11-16T05:58:39,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:39,329 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8a1693b42e3b43998ee96ff71262962d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d 2024-11-16T05:58:39,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d, entries=200, sequenceid=98, filesize=38.6 K 2024-11-16T05:58:39,338 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f32c19e28efc4eedb53bab0264c0a7a6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f32c19e28efc4eedb53bab0264c0a7a6 2024-11-16T05:58:39,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/8d1834e9e99646938153d1231ace8a87 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87 2024-11-16T05:58:39,345 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into f32c19e28efc4eedb53bab0264c0a7a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:39,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:39,345 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=13, startTime=1731736718855; duration=0sec 2024-11-16T05:58:39,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:39,345 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:39,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:58:39,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ddab84ca8c9a4894851a63dbada46e6d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d 2024-11-16T05:58:39,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T05:58:39,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a01be5863776074fa8e10401262fcc00 in 499ms, sequenceid=98, compaction requested=false 2024-11-16T05:58:39,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T05:58:39,443 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:39,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-16T05:58:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:39,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:39,449 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:39,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T05:58:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:39,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:39,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:39,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:39,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111671ea6dab8481441f80cf758bd9155df5_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:39,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742412_1588 (size=12154) 2024-11-16T05:58:39,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111671ea6dab8481441f80cf758bd9155df5_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111671ea6dab8481441f80cf758bd9155df5_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:39,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/e189701793b44ab68e44f6069dc3cc02, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:39,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/e189701793b44ab68e44f6069dc3cc02 is 175, key is test_row_0/A:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:39,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742413_1589 (size=30955) 2024-11-16T05:58:39,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/e189701793b44ab68e44f6069dc3cc02 2024-11-16T05:58:39,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/316aee629deb48e1983ae6442c12d871 is 50, key is test_row_0/B:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:39,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731736719519, max timestamp=1731736719553Num uniques 859; 2024-11-16T05:58:39,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731736719519, max timestamp=1731736719553Num uniques 859; 2024-11-16T05:58:39,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:39,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:39,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731736719519, max timestamp=1731736719553Num uniques 859; 2024-11-16T05:58:39,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T05:58:39,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742414_1590 (size=12001) 2024-11-16T05:58:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736779583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736779584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736779587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-16T05:58:39,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736779693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736779693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736779702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-16T05:58:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:39,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736779773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,777 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_0' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736779783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,786 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736779903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736779903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736779912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:39,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/316aee629deb48e1983ae6442c12d871 2024-11-16T05:58:39,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ab9167af4e9141688029ae99c28b6f3e is 50, key is test_row_0/C:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:39,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742415_1591 (size=12001) 2024-11-16T05:58:39,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ab9167af4e9141688029ae99c28b6f3e 2024-11-16T05:58:39,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/e189701793b44ab68e44f6069dc3cc02 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02 2024-11-16T05:58:39,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02, entries=150, sequenceid=112, filesize=30.2 K 2024-11-16T05:58:39,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/316aee629deb48e1983ae6442c12d871 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871 2024-11-16T05:58:39,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T05:58:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/ab9167af4e9141688029ae99c28b6f3e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e 2024-11-16T05:58:39,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T05:58:39,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a01be5863776074fa8e10401262fcc00 in 490ms, sequenceid=112, compaction requested=true 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:39,984 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:39,984 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:39,985 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,985 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,985 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=99.2 K 2024-11-16T05:58:39,985 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/604acee2c13f41b4bfff94a67e9ff240, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=35.3 K 2024-11-16T05:58:39,985 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02] 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 604acee2c13f41b4bfff94a67e9ff240, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d663168f9e7c4383ae44cb58da6b7511, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1693b42e3b43998ee96ff71262962d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736717682 2024-11-16T05:58:39,985 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1834e9e99646938153d1231ace8a87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736717707 2024-11-16T05:58:39,986 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting e189701793b44ab68e44f6069dc3cc02, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:39,986 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 316aee629deb48e1983ae6442c12d871, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:39,992 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:39,992 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/858e448ee97d43c78707b9fe358756f7 is 50, key is test_row_0/B:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:39,993 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:39,995 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168fd83559f16147ecb499fb704cffe30b_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:39,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742416_1592 (size=12207) 2024-11-16T05:58:39,997 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168fd83559f16147ecb499fb704cffe30b_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:39,997 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168fd83559f16147ecb499fb704cffe30b_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:40,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742417_1593 (size=4469) 2024-11-16T05:58:40,001 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#508 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:40,002 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ac585f9e06d640e69e6ea5d901372287 is 175, key is test_row_0/A:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:40,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742418_1594 (size=31161) 2024-11-16T05:58:40,008 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ac585f9e06d640e69e6ea5d901372287 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287 2024-11-16T05:58:40,012 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into ac585f9e06d640e69e6ea5d901372287(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:40,012 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:40,012 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=13, startTime=1731736719984; duration=0sec 2024-11-16T05:58:40,012 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:40,012 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:40,012 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:40,013 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:40,013 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:40,013 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:40,013 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f32c19e28efc4eedb53bab0264c0a7a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=35.3 K 2024-11-16T05:58:40,013 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting f32c19e28efc4eedb53bab0264c0a7a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731736717480 2024-11-16T05:58:40,014 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddab84ca8c9a4894851a63dbada46e6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731736717707 2024-11-16T05:58:40,014 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9167af4e9141688029ae99c28b6f3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:40,020 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:40,020 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/295519513a7a4fed89bcae8d83c0e9b5 is 50, key is test_row_0/C:col10/1731736719493/Put/seqid=0 2024-11-16T05:58:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742419_1595 (size=12207) 2024-11-16T05:58:40,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-16T05:58:40,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:40,058 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:40,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bfe4222559864dcfa4556d4e0a69c60f_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736719583/Put/seqid=0 2024-11-16T05:58:40,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742420_1596 (size=12304) 2024-11-16T05:58:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:40,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736780229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736780230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736780234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736780343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736780343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736780343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,399 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/858e448ee97d43c78707b9fe358756f7 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/858e448ee97d43c78707b9fe358756f7 2024-11-16T05:58:40,402 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into 858e448ee97d43c78707b9fe358756f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:40,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:40,402 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=13, startTime=1731736719984; duration=0sec 2024-11-16T05:58:40,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:40,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:40,429 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/295519513a7a4fed89bcae8d83c0e9b5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/295519513a7a4fed89bcae8d83c0e9b5 2024-11-16T05:58:40,435 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into 295519513a7a4fed89bcae8d83c0e9b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:40,435 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:40,435 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=13, startTime=1731736719984; duration=0sec 2024-11-16T05:58:40,435 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:40,435 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:40,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bfe4222559864dcfa4556d4e0a69c60f_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bfe4222559864dcfa4556d4e0a69c60f_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:40,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8aa5b9a2e82e43fd9f06eb56e38b7695, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:40,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8aa5b9a2e82e43fd9f06eb56e38b7695 is 175, key is test_row_0/A:col10/1731736719583/Put/seqid=0 2024-11-16T05:58:40,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742421_1597 (size=31105) 2024-11-16T05:58:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736780553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736780553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736780553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736780863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736780863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736780863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:40,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8aa5b9a2e82e43fd9f06eb56e38b7695 2024-11-16T05:58:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f7a71625fdfb45988af1f9ee791e9a66 is 50, key is test_row_0/B:col10/1731736719583/Put/seqid=0 2024-11-16T05:58:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742422_1598 (size=12151) 2024-11-16T05:58:41,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f7a71625fdfb45988af1f9ee791e9a66 2024-11-16T05:58:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/41bf6d39cbdc428a9798426e1fd45509 is 50, key is test_row_0/C:col10/1731736719583/Put/seqid=0 2024-11-16T05:58:41,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742423_1599 (size=12151) 2024-11-16T05:58:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736781373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736781373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736781373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:41,724 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/41bf6d39cbdc428a9798426e1fd45509 2024-11-16T05:58:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/8aa5b9a2e82e43fd9f06eb56e38b7695 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695 2024-11-16T05:58:41,730 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695, entries=150, sequenceid=147, filesize=30.4 K 2024-11-16T05:58:41,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f7a71625fdfb45988af1f9ee791e9a66 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66 2024-11-16T05:58:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T05:58:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/41bf6d39cbdc428a9798426e1fd45509 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509 2024-11-16T05:58:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,740 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T05:58:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for a01be5863776074fa8e10401262fcc00 in 1683ms, sequenceid=147, compaction requested=false 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-16T05:58:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-16T05:58:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-11-16T05:58:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.3020 sec 2024-11-16T05:58:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T05:58:41,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:41,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167cc8e534b450434db890807bf6f92a5f_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:41,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742424_1600 (size=14794) 2024-11-16T05:58:41,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736781868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736781870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736781983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736781983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736782193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736782193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:42,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167cc8e534b450434db890807bf6f92a5f_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cc8e534b450434db890807bf6f92a5f_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:42,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/1070d2b339a14f418fb82f4dd420ecc0, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:42,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/1070d2b339a14f418fb82f4dd420ecc0 is 175, key is test_row_0/A:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742425_1601 (size=39745) 2024-11-16T05:58:42,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/1070d2b339a14f418fb82f4dd420ecc0 2024-11-16T05:58:42,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f240c17476da4f8ba2a3e3875c3440f2 is 50, key is test_row_0/B:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:42,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742426_1602 (size=9757) 2024-11-16T05:58:42,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f240c17476da4f8ba2a3e3875c3440f2 2024-11-16T05:58:42,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/86a13f9f77174a7db147fe70f584ec96 is 50, key is test_row_0/C:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:42,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742427_1603 (size=9757) 2024-11-16T05:58:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736782382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736782382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,385 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:42,385 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736782392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,395 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736782503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736782503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:42,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/86a13f9f77174a7db147fe70f584ec96 2024-11-16T05:58:42,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/1070d2b339a14f418fb82f4dd420ecc0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0 2024-11-16T05:58:42,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0, entries=200, sequenceid=162, filesize=38.8 K 2024-11-16T05:58:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/f240c17476da4f8ba2a3e3875c3440f2 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2 2024-11-16T05:58:42,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2, entries=100, sequenceid=162, filesize=9.5 K 2024-11-16T05:58:42,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/86a13f9f77174a7db147fe70f584ec96 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96 2024-11-16T05:58:42,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96, entries=100, sequenceid=162, filesize=9.5 K 2024-11-16T05:58:42,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a01be5863776074fa8e10401262fcc00 in 899ms, sequenceid=162, compaction requested=true 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:42,713 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:42,713 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:42,714 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:42,714 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:42,714 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/858e448ee97d43c78707b9fe358756f7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=33.3 K 2024-11-16T05:58:42,714 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=99.6 K 2024-11-16T05:58:42,714 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0] 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 858e448ee97d43c78707b9fe358756f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac585f9e06d640e69e6ea5d901372287, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:42,714 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a71625fdfb45988af1f9ee791e9a66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731736719575 2024-11-16T05:58:42,715 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aa5b9a2e82e43fd9f06eb56e38b7695, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731736719575 2024-11-16T05:58:42,715 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f240c17476da4f8ba2a3e3875c3440f2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736721804 2024-11-16T05:58:42,715 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1070d2b339a14f418fb82f4dd420ecc0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736720228 2024-11-16T05:58:42,720 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:42,721 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:42,721 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/0d2d59dafe1349c9bc974023d2e8d552 is 50, key is test_row_0/B:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:42,721 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116eda06433bc4842a1b0f7e9a2b55a25f3_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:42,723 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116eda06433bc4842a1b0f7e9a2b55a25f3_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:42,723 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eda06433bc4842a1b0f7e9a2b55a25f3_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:42,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742429_1605 (size=4469) 2024-11-16T05:58:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742428_1604 (size=12459) 2024-11-16T05:58:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:43,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T05:58:43,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736721867, max timestamp=1731736723014Num uniques 859; 2024-11-16T05:58:43,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736721867, max timestamp=1731736723014Num uniques 859; 2024-11-16T05:58:43,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:43,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:43,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731736721867, max timestamp=1731736723014Num uniques 859; 2024-11-16T05:58:43,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:43,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111641f4da57d39a4b34bb45c4582b7aa89b_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736721867/Put/seqid=0 2024-11-16T05:58:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742430_1606 (size=12304) 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736723044, max timestamp=1731736723064Num uniques 859; 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736723044, max timestamp=1731736723064Num uniques 859; 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731736723044, max timestamp=1731736723064Num uniques 859; 2024-11-16T05:58:43,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T05:58:43,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736723086, max timestamp=1731736723118Num uniques 1068; 2024-11-16T05:58:43,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T05:58:43,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731736723086, max timestamp=1731736723118Num uniques 1068; 2024-11-16T05:58:43,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T05:58:43,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731736723086, max timestamp=1731736723118Num uniques 1068; 2024-11-16T05:58:43,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-16T05:58:43,134 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#517 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:43,134 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ae69fafb5ac246aba6633c1377520107 is 175, key is test_row_0/A:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:43,141 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/0d2d59dafe1349c9bc974023d2e8d552 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/0d2d59dafe1349c9bc974023d2e8d552 2024-11-16T05:58:43,145 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into 0d2d59dafe1349c9bc974023d2e8d552(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:43,145 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:43,145 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=13, startTime=1731736722713; duration=0sec 2024-11-16T05:58:43,145 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:43,145 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:43,145 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:43,146 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:43,146 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:43,146 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:43,146 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/295519513a7a4fed89bcae8d83c0e9b5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=33.3 K 2024-11-16T05:58:43,146 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 295519513a7a4fed89bcae8d83c0e9b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731736718867 2024-11-16T05:58:43,147 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bf6d39cbdc428a9798426e1fd45509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731736719575 2024-11-16T05:58:43,147 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a13f9f77174a7db147fe70f584ec96, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736721804 2024-11-16T05:58:43,152 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:43,153 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/19c2ef195ab347a7b231c775c6eb042c is 50, key is test_row_0/C:col10/1731736721809/Put/seqid=0 2024-11-16T05:58:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742431_1607 (size=31520) 2024-11-16T05:58:43,159 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ae69fafb5ac246aba6633c1377520107 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107 2024-11-16T05:58:43,164 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into ae69fafb5ac246aba6633c1377520107(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:43,164 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:43,164 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=13, startTime=1731736722713; duration=0sec 2024-11-16T05:58:43,164 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:43,164 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742432_1608 (size=12459) 2024-11-16T05:58:43,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736723152, max timestamp=1731736723177Num uniques 1027; 2024-11-16T05:58:43,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T05:58:43,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736723152, max timestamp=1731736723177Num uniques 1027; 2024-11-16T05:58:43,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T05:58:43,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731736723152, max timestamp=1731736723177Num uniques 1009; 2024-11-16T05:58:43,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-16T05:58:43,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736723210, max timestamp=1731736723235Num uniques 1086; 2024-11-16T05:58:43,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:43,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736723210, max timestamp=1731736723235Num uniques 1086; 2024-11-16T05:58:43,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:43,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731736723210, max timestamp=1731736723235Num uniques 1086; 2024-11-16T05:58:43,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731736723271, max timestamp=1731736723288Num uniques 1045; 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731736723271, max timestamp=1731736723288Num uniques 1045; 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731736723271, max timestamp=1731736723288Num uniques 1045; 2024-11-16T05:58:43,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-16T05:58:43,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731736723322, max timestamp=1731736723352Num uniques 1054; 2024-11-16T05:58:43,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-16T05:58:43,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731736723322, max timestamp=1731736723352Num uniques 1054; 2024-11-16T05:58:43,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-16T05:58:43,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731736723322, max timestamp=1731736723352Num uniques 1054; 2024-11-16T05:58:43,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731736723380, max timestamp=1731736723415Num uniques 1063; 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731736723380, max timestamp=1731736723415Num uniques 1063; 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731736723380, max timestamp=1731736723415Num uniques 1063; 2024-11-16T05:58:43,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-16T05:58:43,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:43,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111641f4da57d39a4b34bb45c4582b7aa89b_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111641f4da57d39a4b34bb45c4582b7aa89b_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:43,446 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/778e2bae25e34c7c9c0951da2aa856df, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:43,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/778e2bae25e34c7c9c0951da2aa856df is 175, key is test_row_0/A:col10/1731736721867/Put/seqid=0 2024-11-16T05:58:43,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742433_1609 (size=31105) 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731736723452, max timestamp=1731736723470Num uniques 1022; 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731736723452, max timestamp=1731736723470Num uniques 1022; 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731736723452, max timestamp=1731736723470Num uniques 1022; 2024-11-16T05:58:43,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731736723508, max timestamp=1731736723536Num uniques 1081; 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731736723508, max timestamp=1731736723536Num uniques 1081; 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731736723508, max timestamp=1731736723536Num uniques 1081; 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-16T05:58:43,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-16T05:58:43,575 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/19c2ef195ab347a7b231c775c6eb042c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/19c2ef195ab347a7b231c775c6eb042c 2024-11-16T05:58:43,579 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into 19c2ef195ab347a7b231c775c6eb042c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:43,579 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:43,579 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=13, startTime=1731736722713; duration=0sec 2024-11-16T05:58:43,579 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:43,579 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731736723543, max timestamp=1731736723579Num uniques 1040; 2024-11-16T05:58:43,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-16T05:58:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731736723543, max timestamp=1731736723579Num uniques 1040; 2024-11-16T05:58:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731736723543, max timestamp=1731736723579Num uniques 1040; 2024-11-16T05:58:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-16T05:58:43,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-16T05:58:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-16T05:58:43,603 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:43,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-16T05:58:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:43,605 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:43,606 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:43,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731736723610, max timestamp=1731736723623Num uniques 1099; 2024-11-16T05:58:43,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-16T05:58:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731736723610, max timestamp=1731736723623Num uniques 1049; 2024-11-16T05:58:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-16T05:58:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731736723610, max timestamp=1731736723623Num uniques 1049; 2024-11-16T05:58:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731736723647, max timestamp=1731736723671Num uniques 1108; 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731736723647, max timestamp=1731736723671Num uniques 1108; 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731736723647, max timestamp=1731736723671Num uniques 1108; 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-16T05:58:43,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731736723699, max timestamp=1731736723709Num uniques 1067; 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-16T05:58:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731736723699, max timestamp=1731736723709Num uniques 1067; 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731736723699, max timestamp=1731736723709Num uniques 1067; 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-16T05:58:43,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-16T05:58:43,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731736723716, max timestamp=1731736723752Num uniques 1026; 2024-11-16T05:58:43,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-16T05:58:43,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731736723716, max timestamp=1731736723752Num uniques 1026; 2024-11-16T05:58:43,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-16T05:58:43,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731736723716, max timestamp=1731736723752Num uniques 1026; 2024-11-16T05:58:43,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-16T05:58:43,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:43,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731736723769, max timestamp=1731736723793Num uniques 1085; 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731736723769, max timestamp=1731736723793Num uniques 1085; 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731736723769, max timestamp=1731736723793Num uniques 1085; 2024-11-16T05:58:43,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T05:58:43,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731736723816, max timestamp=1731736723834Num uniques 1044; 2024-11-16T05:58:43,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-16T05:58:43,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731736723816, max timestamp=1731736723834Num uniques 1044; 2024-11-16T05:58:43,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-16T05:58:43,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731736723816, max timestamp=1731736723834Num uniques 1044; 2024-11-16T05:58:43,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-16T05:58:43,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/778e2bae25e34c7c9c0951da2aa856df 2024-11-16T05:58:43,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/58761d8cab8a4030866010a06e53fe2f is 50, key is test_row_0/B:col10/1731736721867/Put/seqid=0 2024-11-16T05:58:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742434_1610 (size=12151) 2024-11-16T05:58:43,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731736723872, max timestamp=1731736723883Num uniques 1103; 2024-11-16T05:58:43,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-16T05:58:43,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731736723872, max timestamp=1731736723883Num uniques 1103; 2024-11-16T05:58:43,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731736723872, max timestamp=1731736723883Num uniques 1103; 2024-11-16T05:58:43,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-16T05:58:43,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-11-16T05:58:43,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:43,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:43,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731736723911, max timestamp=1731736723922Num uniques 1012; 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731736723911, max timestamp=1731736723922Num uniques 1012; 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731736723911, max timestamp=1731736723922Num uniques 1012; 2024-11-16T05:58:43,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731736723952, max timestamp=1731736723974Num uniques 1021; 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731736723952, max timestamp=1731736723974Num uniques 1021; 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731736723952, max timestamp=1731736723974Num uniques 1021; 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-16T05:58:43,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-16T05:58:44,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731736724024, max timestamp=1731736724037Num uniques 1080; 2024-11-16T05:58:44,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-16T05:58:44,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731736724024, max timestamp=1731736724037Num uniques 1080; 2024-11-16T05:58:44,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-16T05:58:44,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731736724024, max timestamp=1731736724037Num uniques 1080; 2024-11-16T05:58:44,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-16T05:58:44,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731736724073, max timestamp=1731736724086Num uniques 1089; 2024-11-16T05:58:44,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-16T05:58:44,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731736724073, max timestamp=1731736724086Num uniques 1089; 2024-11-16T05:58:44,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-16T05:58:44,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731736724073, max timestamp=1731736724086Num uniques 1089; 2024-11-16T05:58:44,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-16T05:58:44,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1731736724111, max timestamp=1731736724126Num uniques 1098; 2024-11-16T05:58:44,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=11358, suffix heap size=142160,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=248 2024-11-16T05:58:44,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1731736724111, max timestamp=1731736724126Num uniques 1098; 2024-11-16T05:58:44,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=11358, suffix heap size=142160,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=248 2024-11-16T05:58:44,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1731736724111, max timestamp=1731736724126Num uniques 1098; 2024-11-16T05:58:44,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=11358, suffix heap size=142160,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=248 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731736724165, max timestamp=1731736724179Num uniques 1107; 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731736724165, max timestamp=1731736724179Num uniques 1107; 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731736724165, max timestamp=1731736724179Num uniques 1107; 2024-11-16T05:58:44,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-16T05:58:44,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:44,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731736724223, max timestamp=1731736724236Num uniques 1066; 2024-11-16T05:58:44,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-16T05:58:44,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731736724223, max timestamp=1731736724236Num uniques 1066; 2024-11-16T05:58:44,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-16T05:58:44,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731736724223, max timestamp=1731736724236Num uniques 1066; 2024-11-16T05:58:44,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-16T05:58:44,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/58761d8cab8a4030866010a06e53fe2f 2024-11-16T05:58:44,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731736724271, max timestamp=1731736724302Num uniques 1025; 2024-11-16T05:58:44,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-16T05:58:44,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731736724271, max timestamp=1731736724302Num uniques 1025; 2024-11-16T05:58:44,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-16T05:58:44,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731736724271, max timestamp=1731736724302Num uniques 1025; 2024-11-16T05:58:44,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-16T05:58:44,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/e98aa508730946708aea6cbc60fb1c8a is 50, key is test_row_0/C:col10/1731736721867/Put/seqid=0 2024-11-16T05:58:44,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742435_1611 (size=12151) 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1731736724303, max timestamp=1731736724350Num uniques 1034; 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1731736724303, max timestamp=1731736724350Num uniques 1034; 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=13004, suffix heap size=135504,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=284 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1731736724303, max timestamp=1731736724350Num uniques 1034; 2024-11-16T05:58:44,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=13004, suffix heap size=135504,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=284 2024-11-16T05:58:44,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1731736724374, max timestamp=1731736724401Num uniques 1093; 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=11128, suffix heap size=141640,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=243 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1731736724374, max timestamp=1731736724401Num uniques 1143; 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=11128, suffix heap size=146840,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=243 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1731736724374, max timestamp=1731736724401Num uniques 1143; 2024-11-16T05:58:44,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=11128, suffix heap size=146840,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=243 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=302, cellSize=13830, totalHeapSize=31648, min timestamp=1731736724421, max timestamp=1731736724430Num uniques 1102; 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=302, cellSize=13830, totalHeapSize=31648, min timestamp=1731736724421, max timestamp=1731736724430Num uniques 1102; 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50470, new segment data size=13830, suffix heap size=142576,new segment heap size=31648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1102, new segment cells count=302 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50470, new segment data size=13830, suffix heap size=142576,new segment heap size=31648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1102, new segment cells count=302 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=302, cellSize=13830, totalHeapSize=31648, min timestamp=1731736724421, max timestamp=1731736724430Num uniques 1102; 2024-11-16T05:58:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50470, new segment data size=13830, suffix heap size=142576,new segment heap size=31648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1102, new segment cells count=302 2024-11-16T05:58:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736784452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736784453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736784453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=211, cellSize=9656, totalHeapSize=22184, min timestamp=1731736724448, max timestamp=1731736724457Num uniques 1161; 2024-11-16T05:58:44,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53166, new segment data size=9656, suffix heap size=148712,new segment heap size=22184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1161, new segment cells count=211 2024-11-16T05:58:44,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=211, cellSize=9656, totalHeapSize=22184, min timestamp=1731736724448, max timestamp=1731736724457Num uniques 1161; 2024-11-16T05:58:44,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53166, new segment data size=9656, suffix heap size=148712,new segment heap size=22184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1161, new segment cells count=211 2024-11-16T05:58:44,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=211, cellSize=9656, totalHeapSize=22184, min timestamp=1731736724448, max timestamp=1731736724457Num uniques 1161; 2024-11-16T05:58:44,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53166, new segment data size=9656, suffix heap size=148712,new segment heap size=22184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1161, new segment cells count=211 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=170, cellSize=7780, totalHeapSize=17920, min timestamp=1731736724487, max timestamp=1731736724503Num uniques 1070; 2024-11-16T05:58:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=170, cellSize=7780, totalHeapSize=17920, min timestamp=1731736724487, max timestamp=1731736724503Num uniques 1070; 2024-11-16T05:58:44,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=170, cellSize=7780, totalHeapSize=17920, min timestamp=1731736724487, max timestamp=1731736724503Num uniques 1070; 2024-11-16T05:58:44,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49000, new segment data size=7780, suffix heap size=139248,new segment heap size=17920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1070, new segment cells count=170 2024-11-16T05:58:44,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49000, new segment data size=7780, suffix heap size=139248,new segment heap size=17920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1070, new segment cells count=170 2024-11-16T05:58:44,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49000, new segment data size=7780, suffix heap size=139248,new segment heap size=17920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1070, new segment cells count=170 2024-11-16T05:58:44,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1731736724527, max timestamp=1731736724549Num uniques 1029; 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1731736724527, max timestamp=1731736724549Num uniques 1029; 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1731736724527, max timestamp=1731736724549Num uniques 1029; 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47124, new segment data size=10484, suffix heap size=134984,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1029, new segment cells count=229 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47124, new segment data size=10484, suffix heap size=134984,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1029, new segment cells count=229 2024-11-16T05:58:44,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47124, new segment data size=10484, suffix heap size=134984,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1029, new segment cells count=229 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731736724565, max timestamp=1731736724584Num uniques 1088; 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731736724565, max timestamp=1731736724584Num uniques 1088; 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=238, cellSize=10898, totalHeapSize=24992, min timestamp=1731736724565, max timestamp=1731736724584Num uniques 1088; 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-16T05:58:44,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=10898, suffix heap size=141120,new segment heap size=24992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=238 2024-11-16T05:58:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736784608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736784609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=247, cellSize=11312, totalHeapSize=25928, min timestamp=1731736724603, max timestamp=1731736724611Num uniques 1097; 2024-11-16T05:58:44,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=11312, suffix heap size=142056,new segment heap size=25928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=247 2024-11-16T05:58:44,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=247, cellSize=11312, totalHeapSize=25928, min timestamp=1731736724603, max timestamp=1731736724611Num uniques 1097; 2024-11-16T05:58:44,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=11312, suffix heap size=142056,new segment heap size=25928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=247 2024-11-16T05:58:44,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1731736724603, max timestamp=1731736724611Num uniques 1097; 2024-11-16T05:58:44,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50242, new segment data size=9022, suffix heap size=142056,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1097, new segment cells count=197 2024-11-16T05:58:44,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9430, totalHeapSize=21664, min timestamp=1731736724630, max timestamp=1731736724640Num uniques 1106; 2024-11-16T05:58:44,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50650, new segment data size=9430, suffix heap size=142992,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1106, new segment cells count=206 2024-11-16T05:58:44,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9430, totalHeapSize=21664, min timestamp=1731736724630, max timestamp=1731736724640Num uniques 1106; 2024-11-16T05:58:44,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50650, new segment data size=9430, suffix heap size=142992,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1106, new segment cells count=206 2024-11-16T05:58:44,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731736724631, max timestamp=1731736724640Num uniques 1056; 2024-11-16T05:58:44,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=6870, suffix heap size=137792,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=150 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7550, totalHeapSize=17400, min timestamp=1731736724662, max timestamp=1731736724668Num uniques 1065; 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48770, new segment data size=7550, suffix heap size=138728,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=165 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7550, totalHeapSize=17400, min timestamp=1731736724662, max timestamp=1731736724668Num uniques 1065; 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48770, new segment data size=7550, suffix heap size=138728,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=165 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=165, cellSize=7550, totalHeapSize=17400, min timestamp=1731736724662, max timestamp=1731736724668Num uniques 1009; 2024-11-16T05:58:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7550, suffix heap size=132904,new segment heap size=17400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=165 2024-11-16T05:58:44,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=224, cellSize=10254, totalHeapSize=23536, min timestamp=1731736724697, max timestamp=1731736724705Num uniques 1024; 2024-11-16T05:58:44,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=224, cellSize=10254, totalHeapSize=23536, min timestamp=1731736724697, max timestamp=1731736724705Num uniques 1024; 2024-11-16T05:58:44,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=10254, suffix heap size=134464,new segment heap size=23536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=224 2024-11-16T05:58:44,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=10254, suffix heap size=134464,new segment heap size=23536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=224 2024-11-16T05:58:44,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1731736724698, max timestamp=1731736724705Num uniques 1024; 2024-11-16T05:58:44,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46894, new segment data size=7964, suffix heap size=134464,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1024, new segment cells count=174 2024-11-16T05:58:44,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/e98aa508730946708aea6cbc60fb1c8a 2024-11-16T05:58:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/778e2bae25e34c7c9c0951da2aa856df as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df 2024-11-16T05:58:44,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df, entries=150, sequenceid=183, filesize=30.4 K 2024-11-16T05:58:44,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/58761d8cab8a4030866010a06e53fe2f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731736724725, max timestamp=1731736724733Num uniques 1083; 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49598, new segment data size=12958, suffix heap size=140600,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1083, new segment cells count=283 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731736724725, max timestamp=1731736724733Num uniques 1083; 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49598, new segment data size=12958, suffix heap size=140600,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1083, new segment cells count=283 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1731736724725, max timestamp=1731736724733Num uniques 1033; 2024-11-16T05:58:44,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3456ee6a3164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=12958, suffix heap size=135400,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=283 2024-11-16T05:58:44,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T05:58:44,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/e98aa508730946708aea6cbc60fb1c8a as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a 2024-11-16T05:58:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:44,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T05:58:44,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=87.22 KB/89310 for a01be5863776074fa8e10401262fcc00 in 1731ms, sequenceid=183, compaction requested=false 2024-11-16T05:58:44,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:44,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=93.93 KB heapSize=253.83 KB 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T05:58:44,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111629c0b450acc9484389e44d015fc989b2_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:44,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742436_1612 (size=12454) 2024-11-16T05:58:44,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736784759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736784760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736784765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736784766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736784766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736784872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736784872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736784883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736784883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736784883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:44,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:44,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736785082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736785082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736785093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736785093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736785093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:45,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111629c0b450acc9484389e44d015fc989b2_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111629c0b450acc9484389e44d015fc989b2_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:45,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4c8de9ac4f904378ade7458aa5efd662, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:45,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4c8de9ac4f904378ade7458aa5efd662 is 175, key is test_row_0/A:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:45,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742437_1613 (size=31255) 2024-11-16T05:58:45,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736785393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736785393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736785403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736785403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736785403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=833, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4c8de9ac4f904378ade7458aa5efd662 2024-11-16T05:58:45,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/c9ce17921d8e4a599875abca68e60f28 is 50, key is test_row_0/B:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:45,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742438_1614 (size=12301) 2024-11-16T05:58:45,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:45,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:45,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736785902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736785902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736785913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736785913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736785913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:45,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/c9ce17921d8e4a599875abca68e60f28 2024-11-16T05:58:45,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/397631fd658440419671781f811aa0dd is 50, key is test_row_0/C:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742439_1615 (size=12301) 2024-11-16T05:58:46,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:46,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/397631fd658440419671781f811aa0dd 2024-11-16T05:58:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/4c8de9ac4f904378ade7458aa5efd662 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662 2024-11-16T05:58:46,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662, entries=150, sequenceid=833, filesize=30.5 K 2024-11-16T05:58:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/c9ce17921d8e4a599875abca68e60f28 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28 2024-11-16T05:58:46,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28, entries=150, sequenceid=833, filesize=12.0 K 2024-11-16T05:58:46,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/397631fd658440419671781f811aa0dd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd 2024-11-16T05:58:46,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd, entries=150, sequenceid=833, filesize=12.0 K 2024-11-16T05:58:46,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~253.78 KB/259872, currentSize=87.22 KB/89310 for a01be5863776074fa8e10401262fcc00 in 1652ms, sequenceid=833, compaction requested=true 2024-11-16T05:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:46,400 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,400 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:46,401 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=91.7 K 2024-11-16T05:58:46,401 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,401 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,401 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662] 2024-11-16T05:58:46,401 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/0d2d59dafe1349c9bc974023d2e8d552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=36.0 K 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae69fafb5ac246aba6633c1377520107, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736719575 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2d59dafe1349c9bc974023d2e8d552, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736719575 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 58761d8cab8a4030866010a06e53fe2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736721867 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 778e2bae25e34c7c9c0951da2aa856df, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736721867 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ce17921d8e4a599875abca68e60f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:46,402 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8de9ac4f904378ade7458aa5efd662, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:46,408 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:46,408 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/a8c9863c230945d5bccb437bb6eab32b is 50, key is test_row_0/B:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:46,409 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742440_1616 (size=12711) 2024-11-16T05:58:46,418 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/a8c9863c230945d5bccb437bb6eab32b as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/a8c9863c230945d5bccb437bb6eab32b 2024-11-16T05:58:46,421 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111655482f8d711d431c9dcff638dbc20eb9_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:46,422 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into a8c9863c230945d5bccb437bb6eab32b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:46,422 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:46,422 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=13, startTime=1731736726400; duration=0sec 2024-11-16T05:58:46,422 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:46,422 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:46,422 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:46,423 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:46,423 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:46,423 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,423 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/19c2ef195ab347a7b231c775c6eb042c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=36.0 K 2024-11-16T05:58:46,423 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c2ef195ab347a7b231c775c6eb042c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731736719575 2024-11-16T05:58:46,424 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111655482f8d711d431c9dcff638dbc20eb9_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:46,424 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting e98aa508730946708aea6cbc60fb1c8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731736721867 2024-11-16T05:58:46,424 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111655482f8d711d431c9dcff638dbc20eb9_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:46,424 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 397631fd658440419671781f811aa0dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:46,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742441_1617 (size=4469) 2024-11-16T05:58:46,443 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:46,443 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c49b71d7f5f042d9bc44254563131a1c is 50, key is test_row_0/C:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742442_1618 (size=12711) 2024-11-16T05:58:46,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-16T05:58:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:46,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:58:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697b80ff5edd7481b8405a6a226cbbdbb_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736724765/Put/seqid=0 2024-11-16T05:58:46,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742443_1619 (size=12454) 2024-11-16T05:58:46,835 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:46,835 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/0e06b1e98d1a4399aa0333e0d55758f9 is 175, key is test_row_0/A:col10/1731736724747/Put/seqid=0 2024-11-16T05:58:46,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742444_1620 (size=31665) 2024-11-16T05:58:46,842 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/0e06b1e98d1a4399aa0333e0d55758f9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9 2024-11-16T05:58:46,845 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into 0e06b1e98d1a4399aa0333e0d55758f9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:46,845 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:46,845 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=13, startTime=1731736726399; duration=0sec 2024-11-16T05:58:46,845 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:46,845 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:46,850 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c49b71d7f5f042d9bc44254563131a1c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c49b71d7f5f042d9bc44254563131a1c 2024-11-16T05:58:46,853 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into c49b71d7f5f042d9bc44254563131a1c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:46,853 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:46,853 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=13, startTime=1731736726400; duration=0sec 2024-11-16T05:58:46,853 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:46,853 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:46,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:46,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:46,929 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697b80ff5edd7481b8405a6a226cbbdbb_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697b80ff5edd7481b8405a6a226cbbdbb_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:46,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/15c3d0980b2348fca5c9145208760bfd, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/15c3d0980b2348fca5c9145208760bfd is 175, key is test_row_0/A:col10/1731736724765/Put/seqid=0 2024-11-16T05:58:46,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742445_1621 (size=31255) 2024-11-16T05:58:46,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736786931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736786932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736786933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736786933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:46,937 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:46,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736786935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736787043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736787043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736787043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736787043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736787253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736787253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736787253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736787253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=850, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/15c3d0980b2348fca5c9145208760bfd 2024-11-16T05:58:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/fe3cb9a3fcb9442fb51d2ad269bf0663 is 50, key is test_row_0/B:col10/1731736724765/Put/seqid=0 2024-11-16T05:58:47,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742446_1622 (size=12301) 2024-11-16T05:58:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736787562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736787563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736787563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736787563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:47,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=850 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/fe3cb9a3fcb9442fb51d2ad269bf0663 2024-11-16T05:58:47,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c823374827c4458fbf4202d1eac89d08 is 50, key is test_row_0/C:col10/1731736724765/Put/seqid=0 2024-11-16T05:58:47,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742447_1623 (size=12301) 2024-11-16T05:58:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736788073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736788073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736788073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736788073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:48,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=850 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c823374827c4458fbf4202d1eac89d08 2024-11-16T05:58:48,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/15c3d0980b2348fca5c9145208760bfd as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd 2024-11-16T05:58:48,160 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd, entries=150, sequenceid=850, filesize=30.5 K 2024-11-16T05:58:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/fe3cb9a3fcb9442fb51d2ad269bf0663 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663 2024-11-16T05:58:48,163 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663, entries=150, sequenceid=850, filesize=12.0 K 2024-11-16T05:58:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c823374827c4458fbf4202d1eac89d08 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08 2024-11-16T05:58:48,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08, entries=150, sequenceid=850, filesize=12.0 K 2024-11-16T05:58:48,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a01be5863776074fa8e10401262fcc00 in 1652ms, sequenceid=850, compaction requested=false 2024-11-16T05:58:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-16T05:58:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-16T05:58:48,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T05:58:48,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5620 sec 2024-11-16T05:58:48,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 4.5680 sec 2024-11-16T05:58:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:48,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:48,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:48,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cb11f5a340d2484382a6a76330125995_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:48,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742448_1624 (size=14994) 2024-11-16T05:58:48,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:48,971 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cb11f5a340d2484382a6a76330125995_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb11f5a340d2484382a6a76330125995_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:48,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/700836597d26451b9cc1cebdfc3365a6, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:48,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/700836597d26451b9cc1cebdfc3365a6 is 175, key is test_row_0/A:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742449_1625 (size=39949) 2024-11-16T05:58:48,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=869, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/700836597d26451b9cc1cebdfc3365a6 2024-11-16T05:58:48,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/9e1161153dcd4f8d8bbe18879fe91321 is 50, key is test_row_0/B:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:48,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742450_1626 (size=12301) 2024-11-16T05:58:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736789014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736789083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736789083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,088 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:49,088 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736789093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736789093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,097 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,098 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736789123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736789333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=869 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/9e1161153dcd4f8d8bbe18879fe91321 2024-11-16T05:58:49,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/8a458185ce644f85b63425b526b3610f is 50, key is test_row_0/C:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742451_1627 (size=12301) 2024-11-16T05:58:49,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736789642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:49,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=869 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/8a458185ce644f85b63425b526b3610f 2024-11-16T05:58:49,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/700836597d26451b9cc1cebdfc3365a6 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6 2024-11-16T05:58:49,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6, entries=200, sequenceid=869, filesize=39.0 K 2024-11-16T05:58:49,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/9e1161153dcd4f8d8bbe18879fe91321 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321 2024-11-16T05:58:49,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321, entries=150, sequenceid=869, filesize=12.0 K 2024-11-16T05:58:49,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/8a458185ce644f85b63425b526b3610f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f 2024-11-16T05:58:49,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f, entries=150, sequenceid=869, filesize=12.0 K 2024-11-16T05:58:49,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a01be5863776074fa8e10401262fcc00 in 860ms, sequenceid=869, compaction requested=true 2024-11-16T05:58:49,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:49,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:49,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:49,817 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:49,817 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:49,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:49,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:49,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:49,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:49,825 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:49,825 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:49,825 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:49,825 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:49,825 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:49,825 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=100.5 K 2024-11-16T05:58:49,825 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:49,825 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:49,825 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/a8c9863c230945d5bccb437bb6eab32b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=36.4 K 2024-11-16T05:58:49,825 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6] 2024-11-16T05:58:49,826 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c9863c230945d5bccb437bb6eab32b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:49,826 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e06b1e98d1a4399aa0333e0d55758f9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:49,826 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c3d0980b2348fca5c9145208760bfd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731736724749 2024-11-16T05:58:49,827 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3cb9a3fcb9442fb51d2ad269bf0663, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731736724749 2024-11-16T05:58:49,827 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 700836597d26451b9cc1cebdfc3365a6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:49,827 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1161153dcd4f8d8bbe18879fe91321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:49,832 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:49,832 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:49,833 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/9512ab54c2234627bfd9c2b9d98613e0 is 50, key is test_row_0/B:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:49,834 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116fe9ed9b969064b9facfddd9a732f08a5_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:49,836 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116fe9ed9b969064b9facfddd9a732f08a5_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:49,836 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fe9ed9b969064b9facfddd9a732f08a5_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:49,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742453_1629 (size=4469) 2024-11-16T05:58:49,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742452_1628 (size=12813) 2024-11-16T05:58:49,849 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/9512ab54c2234627bfd9c2b9d98613e0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9512ab54c2234627bfd9c2b9d98613e0 2024-11-16T05:58:49,853 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into 9512ab54c2234627bfd9c2b9d98613e0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:49,853 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:49,853 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=13, startTime=1731736729817; duration=0sec 2024-11-16T05:58:49,853 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:49,853 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:49,853 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T05:58:49,854 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T05:58:49,854 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:49,854 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:49,854 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c49b71d7f5f042d9bc44254563131a1c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=36.4 K 2024-11-16T05:58:49,854 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49b71d7f5f042d9bc44254563131a1c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731736724741 2024-11-16T05:58:49,854 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting c823374827c4458fbf4202d1eac89d08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731736724749 2024-11-16T05:58:49,855 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a458185ce644f85b63425b526b3610f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:49,860 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:49,861 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c51dfed79b7e4f6b9f84512bba1d554d is 50, key is test_row_0/C:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742454_1630 (size=12813) 2024-11-16T05:58:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:50,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:58:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:50,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:50,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:50,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:50,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:50,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:50,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111672cba8b36f064e7b8a721e68c8d36d4e_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736730153/Put/seqid=0 2024-11-16T05:58:50,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742455_1631 (size=12454) 2024-11-16T05:58:50,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736790215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:50,242 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#535 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:50,243 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/d21924e6e6a249d7b8853bfbb31ef875 is 175, key is test_row_0/A:col10/1731736726923/Put/seqid=0 2024-11-16T05:58:50,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742456_1632 (size=31767) 2024-11-16T05:58:50,255 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/d21924e6e6a249d7b8853bfbb31ef875 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875 2024-11-16T05:58:50,259 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into d21924e6e6a249d7b8853bfbb31ef875(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:50,259 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:50,259 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=13, startTime=1731736729816; duration=0sec 2024-11-16T05:58:50,260 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:50,260 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:50,269 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c51dfed79b7e4f6b9f84512bba1d554d as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c51dfed79b7e4f6b9f84512bba1d554d 2024-11-16T05:58:50,274 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into c51dfed79b7e4f6b9f84512bba1d554d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:50,274 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:50,274 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=13, startTime=1731736729817; duration=0sec 2024-11-16T05:58:50,274 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:50,274 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:50,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736790323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:50,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736790532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:50,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:50,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111672cba8b36f064e7b8a721e68c8d36d4e_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111672cba8b36f064e7b8a721e68c8d36d4e_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:50,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b40113512e264f6b865d7183523c4b0c, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:50,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b40113512e264f6b865d7183523c4b0c is 175, key is test_row_0/A:col10/1731736730153/Put/seqid=0 2024-11-16T05:58:50,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742457_1633 (size=31255) 2024-11-16T05:58:50,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736790843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:50,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=886, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b40113512e264f6b865d7183523c4b0c 2024-11-16T05:58:50,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/ce3ce58e8d924852a37cc54d9e802160 is 50, key is test_row_0/B:col10/1731736730153/Put/seqid=0 2024-11-16T05:58:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742458_1634 (size=12301) 2024-11-16T05:58:50,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/ce3ce58e8d924852a37cc54d9e802160 2024-11-16T05:58:50,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c5b28c868633434ba138a1d7e37958b0 is 50, key is test_row_0/C:col10/1731736730153/Put/seqid=0 2024-11-16T05:58:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742459_1635 (size=12301) 2024-11-16T05:58:51,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736791093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:51,096 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736791102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:51,106 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736791112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:51,116 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:51,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736791123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:51,125 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:51,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:51,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736791352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:51,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c5b28c868633434ba138a1d7e37958b0 2024-11-16T05:58:51,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/b40113512e264f6b865d7183523c4b0c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c 2024-11-16T05:58:51,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c, entries=150, sequenceid=886, filesize=30.5 K 2024-11-16T05:58:51,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/ce3ce58e8d924852a37cc54d9e802160 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160 2024-11-16T05:58:51,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160, entries=150, sequenceid=886, filesize=12.0 K 2024-11-16T05:58:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/c5b28c868633434ba138a1d7e37958b0 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0 2024-11-16T05:58:51,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0, entries=150, sequenceid=886, filesize=12.0 K 2024-11-16T05:58:51,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a01be5863776074fa8e10401262fcc00 in 1248ms, sequenceid=886, compaction requested=false 2024-11-16T05:58:51,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T05:58:51,803 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:51,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-16T05:58:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:51,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:51,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:51,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:51,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T05:58:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:51,957 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:51,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164396c29a933147c78b950eb4257791a8_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736730214/Put/seqid=0 2024-11-16T05:58:51,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742460_1636 (size=12454) 2024-11-16T05:58:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:51,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164396c29a933147c78b950eb4257791a8_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164396c29a933147c78b950eb4257791a8_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/203039deb4684126b32960d6fbd7a495, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/203039deb4684126b32960d6fbd7a495 is 175, key is test_row_0/A:col10/1731736730214/Put/seqid=0 2024-11-16T05:58:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742461_1637 (size=31255) 2024-11-16T05:58:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:52,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:52,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=904, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/203039deb4684126b32960d6fbd7a495 2024-11-16T05:58:52,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1314bc4f197442ea89d1fae545376c4e is 50, key is test_row_0/B:col10/1731736730214/Put/seqid=0 2024-11-16T05:58:52,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742462_1638 (size=12301) 2024-11-16T05:58:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736792419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736792533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736792743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:52,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=904 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1314bc4f197442ea89d1fae545376c4e 2024-11-16T05:58:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f8e17e5c46fa4e2d88183633ff6812e9 is 50, key is test_row_0/C:col10/1731736730214/Put/seqid=0 2024-11-16T05:58:52,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742463_1639 (size=12301) 2024-11-16T05:58:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:53,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736793053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:53,193 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=904 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f8e17e5c46fa4e2d88183633ff6812e9 2024-11-16T05:58:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/203039deb4684126b32960d6fbd7a495 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495 2024-11-16T05:58:53,200 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495, entries=150, sequenceid=904, filesize=30.5 K 2024-11-16T05:58:53,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1314bc4f197442ea89d1fae545376c4e as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e 2024-11-16T05:58:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e, entries=150, sequenceid=904, filesize=12.0 K 2024-11-16T05:58:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f8e17e5c46fa4e2d88183633ff6812e9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9 2024-11-16T05:58:53,206 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9, entries=150, sequenceid=904, filesize=12.0 K 2024-11-16T05:58:53,206 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a01be5863776074fa8e10401262fcc00 in 1249ms, sequenceid=904, compaction requested=true 2024-11-16T05:58:53,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:53,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:53,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-16T05:58:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-16T05:58:53,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-16T05:58:53,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-16T05:58:53,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4060 sec 2024-11-16T05:58:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:53,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:53,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:53,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164130d49ce3e649d7997d8737e9c11da8_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:53,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742464_1640 (size=14994) 2024-11-16T05:58:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736793625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736793733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:53,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736793943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T05:58:53,953 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:58:53,953 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T05:58:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-16T05:58:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:53,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T05:58:53,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T05:58:53,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T05:58:53,982 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:53,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164130d49ce3e649d7997d8737e9c11da8_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164130d49ce3e649d7997d8737e9c11da8_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:53,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/94ae6489f2494c8c8d4eeeb1bba683f1, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:53,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/94ae6489f2494c8c8d4eeeb1bba683f1 is 175, key is test_row_0/A:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:53,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742465_1641 (size=39949) 2024-11-16T05:58:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:54,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736794253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:54,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:54,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=921, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/94ae6489f2494c8c8d4eeeb1bba683f1 2024-11-16T05:58:54,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/b464395189ab409a8c6f66b5f47f23f5 is 50, key is test_row_0/B:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:54,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742466_1642 (size=12301) 2024-11-16T05:58:54,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:54,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48132 deadline: 1731736794763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:54,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=921 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/b464395189ab409a8c6f66b5f47f23f5 2024-11-16T05:58:54,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/08ce259b729f4dc4b1d42dbf1e5a3123 is 50, key is test_row_0/C:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742467_1643 (size=12301) 2024-11-16T05:58:54,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:54,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:54,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:54,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:55,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48116 deadline: 1731736795123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:55,125 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:55,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48148 deadline: 1731736795133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:55,136 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731736795142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:55,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:58:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45735 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48176 deadline: 1731736795142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 2024-11-16T05:58:55,146 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_1' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:55,146 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,45735,1731736558702 for 'test_row_2' in a01be5863776074fa8e10401262fcc00 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 , the old value is region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:58:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., hostname=3456ee6a3164,45735,1731736558702, seqNum=5 because the exception is null or not the one we care about 2024-11-16T05:58:55,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:58:55,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=921 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/08ce259b729f4dc4b1d42dbf1e5a3123 2024-11-16T05:58:55,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/94ae6489f2494c8c8d4eeeb1bba683f1 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1 2024-11-16T05:58:55,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1, entries=200, sequenceid=921, filesize=39.0 K 2024-11-16T05:58:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/b464395189ab409a8c6f66b5f47f23f5 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5 2024-11-16T05:58:55,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5, entries=150, sequenceid=921, filesize=12.0 K 2024-11-16T05:58:55,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/08ce259b729f4dc4b1d42dbf1e5a3123 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123 2024-11-16T05:58:55,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123, entries=150, sequenceid=921, filesize=12.0 K 2024-11-16T05:58:55,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a01be5863776074fa8e10401262fcc00 in 1654ms, sequenceid=921, compaction requested=true 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T05:58:55,219 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:55,219 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a01be5863776074fa8e10401262fcc00:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T05:58:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:55,220 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:55,220 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:55,220 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/B is initiating minor compaction (all files) 2024-11-16T05:58:55,220 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/A is initiating minor compaction (all files) 2024-11-16T05:58:55,220 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/B in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,220 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/A in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,221 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9512ab54c2234627bfd9c2b9d98613e0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=48.6 K 2024-11-16T05:58:55,221 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=131.1 K 2024-11-16T05:58:55,221 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. files: [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1] 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 9512ab54c2234627bfd9c2b9d98613e0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting d21924e6e6a249d7b8853bfbb31ef875, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3ce58e8d924852a37cc54d9e802160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731736729006 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 1314bc4f197442ea89d1fae545376c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=904, earliestPutTs=1731736730193 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40113512e264f6b865d7183523c4b0c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731736729006 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting b464395189ab409a8c6f66b5f47f23f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=921, earliestPutTs=1731736732408 2024-11-16T05:58:55,221 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203039deb4684126b32960d6fbd7a495, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=904, earliestPutTs=1731736730193 2024-11-16T05:58:55,222 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ae6489f2494c8c8d4eeeb1bba683f1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=921, earliestPutTs=1731736732408 2024-11-16T05:58:55,229 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#B#compaction#546 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:55,230 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1a66fab5be8f47759829b7866e705f10 is 50, key is test_row_0/B:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:55,231 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:55,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742468_1644 (size=12949) 2024-11-16T05:58:55,238 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111643c177c1e0a4424a94ccf1b6f3b28219_a01be5863776074fa8e10401262fcc00 store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:55,240 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111643c177c1e0a4424a94ccf1b6f3b28219_a01be5863776074fa8e10401262fcc00, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:55,241 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111643c177c1e0a4424a94ccf1b6f3b28219_a01be5863776074fa8e10401262fcc00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742469_1645 (size=4469) 2024-11-16T05:58:55,279 INFO [Thread-2529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2529. 2024-11-16T05:58:55,279 DEBUG [Thread-2529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,279 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,280 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:55,280 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-11-16T05:58:55,280 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,280 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,280 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:55,281 INFO [Thread-2527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2527. 2024-11-16T05:58:55,281 DEBUG [Thread-2527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,281 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,281 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:55,282 INFO [Thread-2525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2525. 2024-11-16T05:58:55,282 DEBUG [Thread-2525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,282 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,282 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:55,284 INFO [Thread-2523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2523. 2024-11-16T05:58:55,284 DEBUG [Thread-2523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,284 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,284 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:55,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45735 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T05:58:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T05:58:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:58:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:58:55,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:58:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:58:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111628c04426f1434a6b8c1c713248ea6adf_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_0/A:col10/1731736733605/Put/seqid=0 2024-11-16T05:58:55,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742470_1646 (size=12454) 2024-11-16T05:58:55,645 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1a66fab5be8f47759829b7866e705f10 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1a66fab5be8f47759829b7866e705f10 2024-11-16T05:58:55,648 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#A#compaction#547 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:55,649 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/245d0c8d570f447a8dabf9d5e27722d9 is 175, key is test_row_0/A:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:55,653 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a01be5863776074fa8e10401262fcc00/B of a01be5863776074fa8e10401262fcc00 into 1a66fab5be8f47759829b7866e705f10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:55,653 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:55,653 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/B, priority=12, startTime=1731736735219; duration=0sec 2024-11-16T05:58:55,653 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T05:58:55,653 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:B 2024-11-16T05:58:55,653 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T05:58:55,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742471_1647 (size=31903) 2024-11-16T05:58:55,654 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T05:58:55,654 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1541): a01be5863776074fa8e10401262fcc00/C is initiating minor compaction (all files) 2024-11-16T05:58:55,654 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a01be5863776074fa8e10401262fcc00/C in TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:55,654 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c51dfed79b7e4f6b9f84512bba1d554d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123] into tmpdir=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp, totalSize=48.6 K 2024-11-16T05:58:55,655 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c51dfed79b7e4f6b9f84512bba1d554d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1731736726923 2024-11-16T05:58:55,655 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b28c868633434ba138a1d7e37958b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731736729006 2024-11-16T05:58:55,655 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e17e5c46fa4e2d88183633ff6812e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=904, earliestPutTs=1731736730193 2024-11-16T05:58:55,655 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ce259b729f4dc4b1d42dbf1e5a3123, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=921, earliestPutTs=1731736732408 2024-11-16T05:58:55,661 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a01be5863776074fa8e10401262fcc00#C#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T05:58:55,662 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f6b91b2b7a484b2b8ad8cf436d2a0b1f is 50, key is test_row_0/C:col10/1731736733564/Put/seqid=0 2024-11-16T05:58:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742472_1648 (size=12949) 2024-11-16T05:58:55,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:58:55,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111628c04426f1434a6b8c1c713248ea6adf_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628c04426f1434a6b8c1c713248ea6adf_a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:55,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/27d4b5ebb23c4760998b0dbf8139e334, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:58:55,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/27d4b5ebb23c4760998b0dbf8139e334 is 175, key is test_row_0/A:col10/1731736733605/Put/seqid=0 2024-11-16T05:58:55,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742473_1649 (size=31255) 2024-11-16T05:58:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45735 {}] regionserver.HRegion(8855): Flush requested on a01be5863776074fa8e10401262fcc00 2024-11-16T05:58:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. as already flushing 2024-11-16T05:58:55,776 INFO [Thread-2512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2512. 2024-11-16T05:58:55,776 DEBUG [Thread-2512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:58:55,776 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,776 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:58:55,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:58:56,058 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/245d0c8d570f447a8dabf9d5e27722d9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/245d0c8d570f447a8dabf9d5e27722d9 2024-11-16T05:58:56,061 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a01be5863776074fa8e10401262fcc00/A of a01be5863776074fa8e10401262fcc00 into 245d0c8d570f447a8dabf9d5e27722d9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:56,061 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:56,061 INFO [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/A, priority=12, startTime=1731736735219; duration=0sec 2024-11-16T05:58:56,061 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:56,061 DEBUG [RS:0;3456ee6a3164:45735-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:A 2024-11-16T05:58:56,068 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/f6b91b2b7a484b2b8ad8cf436d2a0b1f as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f6b91b2b7a484b2b8ad8cf436d2a0b1f 2024-11-16T05:58:56,071 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a01be5863776074fa8e10401262fcc00/C of a01be5863776074fa8e10401262fcc00 into f6b91b2b7a484b2b8ad8cf436d2a0b1f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T05:58:56,071 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:56,071 INFO [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., storeName=a01be5863776074fa8e10401262fcc00/C, priority=12, startTime=1731736735219; duration=0sec 2024-11-16T05:58:56,071 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T05:58:56,072 DEBUG [RS:0;3456ee6a3164:45735-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a01be5863776074fa8e10401262fcc00:C 2024-11-16T05:58:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:56,153 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=936, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/27d4b5ebb23c4760998b0dbf8139e334 2024-11-16T05:58:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1b709c2a36f6494cb60ff4ae6579aafa is 50, key is test_row_0/B:col10/1731736733605/Put/seqid=0 2024-11-16T05:58:56,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742474_1650 (size=12301) 2024-11-16T05:58:56,562 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=936 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1b709c2a36f6494cb60ff4ae6579aafa 2024-11-16T05:58:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/26143c64bd5c4f35b32484796dafd1c3 is 50, key is test_row_0/C:col10/1731736733605/Put/seqid=0 2024-11-16T05:58:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742475_1651 (size=12301) 2024-11-16T05:58:56,979 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=936 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/26143c64bd5c4f35b32484796dafd1c3 2024-11-16T05:58:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/27d4b5ebb23c4760998b0dbf8139e334 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/27d4b5ebb23c4760998b0dbf8139e334 2024-11-16T05:58:56,985 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/27d4b5ebb23c4760998b0dbf8139e334, entries=150, sequenceid=936, filesize=30.5 K 2024-11-16T05:58:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/1b709c2a36f6494cb60ff4ae6579aafa as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1b709c2a36f6494cb60ff4ae6579aafa 2024-11-16T05:58:56,988 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1b709c2a36f6494cb60ff4ae6579aafa, entries=150, sequenceid=936, filesize=12.0 K 2024-11-16T05:58:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/26143c64bd5c4f35b32484796dafd1c3 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/26143c64bd5c4f35b32484796dafd1c3 2024-11-16T05:58:56,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/26143c64bd5c4f35b32484796dafd1c3, entries=150, sequenceid=936, filesize=12.0 K 2024-11-16T05:58:56,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for a01be5863776074fa8e10401262fcc00 in 1659ms, sequenceid=936, compaction requested=false 2024-11-16T05:58:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for a01be5863776074fa8e10401262fcc00: 2024-11-16T05:58:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:58:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-16T05:58:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-16T05:58:56,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T05:58:56,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0360 sec 2024-11-16T05:58:56,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0410 sec 2024-11-16T05:58:57,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:58:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T05:58:58,114 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:59:05,156 INFO [Thread-2520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2520. 2024-11-16T05:59:05,157 DEBUG [Thread-2520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:59:05,157 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,157 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:05,195 INFO [Thread-2514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2514. 2024-11-16T05:59:05,195 DEBUG [Thread-2514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:59:05,195 DEBUG [Thread-2514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,195 DEBUG [Thread-2514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:05,216 INFO [Thread-2518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2518. 2024-11-16T05:59:05,216 INFO [Thread-2516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2516. 2024-11-16T05:59:05,216 DEBUG [Thread-2518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:59:05,216 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,216 DEBUG [Thread-2516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T05:59:05,216 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,216 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,216 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:05,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 303 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 280 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2988 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2978 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2919 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2984 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2987 2024-11-16T05:59:05,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T05:59:05,218 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:59:05,218 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:59:05,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:05,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:05,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T05:59:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:05,224 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736745224"}]},"ts":"1731736745224"} 2024-11-16T05:59:05,226 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T05:59:05,226 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T05:59:05,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T05:59:05,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, UNASSIGN}] 2024-11-16T05:59:05,229 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, UNASSIGN 2024-11-16T05:59:05,230 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=CLOSING, regionLocation=3456ee6a3164,45735,1731736558702 2024-11-16T05:59:05,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, UNASSIGN because future has completed 2024-11-16T05:59:05,232 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:59:05,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702}] 2024-11-16T05:59:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:05,384 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:05,384 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:59:05,384 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing a01be5863776074fa8e10401262fcc00, disabling compactions & flushes 2024-11-16T05:59:05,384 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:59:05,384 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:59:05,384 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. after waiting 0 ms 2024-11-16T05:59:05,384 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:59:05,385 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing a01be5863776074fa8e10401262fcc00 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=A 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=B 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a01be5863776074fa8e10401262fcc00, store=C 2024-11-16T05:59:05,385 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T05:59:05,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e37751c4c05340aeaf8dee7d8290e1d7_a01be5863776074fa8e10401262fcc00 is 50, key is test_row_1/A:col10/1731736745213/Put/seqid=0 2024-11-16T05:59:05,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742476_1652 (size=9914) 2024-11-16T05:59:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:05,796 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:59:05,806 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e37751c4c05340aeaf8dee7d8290e1d7_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e37751c4c05340aeaf8dee7d8290e1d7_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:05,807 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ee5b05cabed848448ef19b37efcd1c38, store: [table=TestAcidGuarantees family=A region=a01be5863776074fa8e10401262fcc00] 2024-11-16T05:59:05,808 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ee5b05cabed848448ef19b37efcd1c38 is 175, key is test_row_1/A:col10/1731736745213/Put/seqid=0 2024-11-16T05:59:05,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742477_1653 (size=22561) 2024-11-16T05:59:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:06,213 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=947, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ee5b05cabed848448ef19b37efcd1c38 2024-11-16T05:59:06,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/d8d82a6a5fb94803baffe95ab58e218c is 50, key is test_row_1/B:col10/1731736745213/Put/seqid=0 2024-11-16T05:59:06,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742478_1654 (size=9857) 2024-11-16T05:59:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:06,629 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=947 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/d8d82a6a5fb94803baffe95ab58e218c 2024-11-16T05:59:06,641 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/a89aac7e452a466585fbdcbfa9df75ff is 50, key is test_row_1/C:col10/1731736745213/Put/seqid=0 2024-11-16T05:59:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742479_1655 (size=9857) 2024-11-16T05:59:07,045 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=947 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/a89aac7e452a466585fbdcbfa9df75ff 2024-11-16T05:59:07,051 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/A/ee5b05cabed848448ef19b37efcd1c38 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ee5b05cabed848448ef19b37efcd1c38 2024-11-16T05:59:07,056 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ee5b05cabed848448ef19b37efcd1c38, entries=100, sequenceid=947, filesize=22.0 K 2024-11-16T05:59:07,057 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/B/d8d82a6a5fb94803baffe95ab58e218c as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/d8d82a6a5fb94803baffe95ab58e218c 2024-11-16T05:59:07,062 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/d8d82a6a5fb94803baffe95ab58e218c, entries=100, sequenceid=947, filesize=9.6 K 2024-11-16T05:59:07,063 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/.tmp/C/a89aac7e452a466585fbdcbfa9df75ff as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/a89aac7e452a466585fbdcbfa9df75ff 2024-11-16T05:59:07,068 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/a89aac7e452a466585fbdcbfa9df75ff, entries=100, sequenceid=947, filesize=9.6 K 2024-11-16T05:59:07,070 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a01be5863776074fa8e10401262fcc00 in 1685ms, sequenceid=947, compaction requested=true 2024-11-16T05:59:07,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1] to archive 2024-11-16T05:59:07,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:59:07,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4fa19b5983584b99b5e9d624c590d9ad 2024-11-16T05:59:07,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/6b259b868a7c43e6959beefd43ee48ae 2024-11-16T05:59:07,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d663168f9e7c4383ae44cb58da6b7511 2024-11-16T05:59:07,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b5f79ffc67254edc8d2ca14ddb0d44cc 2024-11-16T05:59:07,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8a1693b42e3b43998ee96ff71262962d 2024-11-16T05:59:07,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ac585f9e06d640e69e6ea5d901372287 2024-11-16T05:59:07,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/e189701793b44ab68e44f6069dc3cc02 2024-11-16T05:59:07,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/8aa5b9a2e82e43fd9f06eb56e38b7695 2024-11-16T05:59:07,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/1070d2b339a14f418fb82f4dd420ecc0 2024-11-16T05:59:07,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ae69fafb5ac246aba6633c1377520107 2024-11-16T05:59:07,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/778e2bae25e34c7c9c0951da2aa856df 2024-11-16T05:59:07,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/0e06b1e98d1a4399aa0333e0d55758f9 2024-11-16T05:59:07,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/4c8de9ac4f904378ade7458aa5efd662 2024-11-16T05:59:07,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/15c3d0980b2348fca5c9145208760bfd 2024-11-16T05:59:07,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/700836597d26451b9cc1cebdfc3365a6 2024-11-16T05:59:07,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/d21924e6e6a249d7b8853bfbb31ef875 2024-11-16T05:59:07,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/b40113512e264f6b865d7183523c4b0c 2024-11-16T05:59:07,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/203039deb4684126b32960d6fbd7a495 2024-11-16T05:59:07,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/94ae6489f2494c8c8d4eeeb1bba683f1 2024-11-16T05:59:07,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/604acee2c13f41b4bfff94a67e9ff240, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/858e448ee97d43c78707b9fe358756f7, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/0d2d59dafe1349c9bc974023d2e8d552, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/a8c9863c230945d5bccb437bb6eab32b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9512ab54c2234627bfd9c2b9d98613e0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5] to archive 2024-11-16T05:59:07,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:59:07,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/af9c0319c92f48638321b4bc05a89451 2024-11-16T05:59:07,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/6bb33f6de40c49689d0b7645b31e5ed5 2024-11-16T05:59:07,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/604acee2c13f41b4bfff94a67e9ff240 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/604acee2c13f41b4bfff94a67e9ff240 2024-11-16T05:59:07,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/35862b8b9d2d4638991077c736962293 2024-11-16T05:59:07,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/8d1834e9e99646938153d1231ace8a87 2024-11-16T05:59:07,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/858e448ee97d43c78707b9fe358756f7 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/858e448ee97d43c78707b9fe358756f7 2024-11-16T05:59:07,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/316aee629deb48e1983ae6442c12d871 2024-11-16T05:59:07,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f7a71625fdfb45988af1f9ee791e9a66 2024-11-16T05:59:07,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/0d2d59dafe1349c9bc974023d2e8d552 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/0d2d59dafe1349c9bc974023d2e8d552 2024-11-16T05:59:07,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/f240c17476da4f8ba2a3e3875c3440f2 2024-11-16T05:59:07,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/58761d8cab8a4030866010a06e53fe2f 2024-11-16T05:59:07,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/a8c9863c230945d5bccb437bb6eab32b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/a8c9863c230945d5bccb437bb6eab32b 2024-11-16T05:59:07,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/c9ce17921d8e4a599875abca68e60f28 2024-11-16T05:59:07,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/fe3cb9a3fcb9442fb51d2ad269bf0663 2024-11-16T05:59:07,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9512ab54c2234627bfd9c2b9d98613e0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9512ab54c2234627bfd9c2b9d98613e0 2024-11-16T05:59:07,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/9e1161153dcd4f8d8bbe18879fe91321 2024-11-16T05:59:07,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/ce3ce58e8d924852a37cc54d9e802160 2024-11-16T05:59:07,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1314bc4f197442ea89d1fae545376c4e 2024-11-16T05:59:07,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/b464395189ab409a8c6f66b5f47f23f5 2024-11-16T05:59:07,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f32c19e28efc4eedb53bab0264c0a7a6, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/295519513a7a4fed89bcae8d83c0e9b5, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/19c2ef195ab347a7b231c775c6eb042c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c49b71d7f5f042d9bc44254563131a1c, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c51dfed79b7e4f6b9f84512bba1d554d, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123] to archive 2024-11-16T05:59:07,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T05:59:07,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/34703b03f03d464a8e9f6be8bcd4be9b 2024-11-16T05:59:07,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/de58600ce0ab4f3282d7e41258f8b8ea 2024-11-16T05:59:07,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f32c19e28efc4eedb53bab0264c0a7a6 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f32c19e28efc4eedb53bab0264c0a7a6 2024-11-16T05:59:07,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/b91f8a49c3b3441e8040edd54e908aa8 2024-11-16T05:59:07,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ddab84ca8c9a4894851a63dbada46e6d 2024-11-16T05:59:07,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/295519513a7a4fed89bcae8d83c0e9b5 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/295519513a7a4fed89bcae8d83c0e9b5 2024-11-16T05:59:07,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/ab9167af4e9141688029ae99c28b6f3e 2024-11-16T05:59:07,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/41bf6d39cbdc428a9798426e1fd45509 2024-11-16T05:59:07,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/19c2ef195ab347a7b231c775c6eb042c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/19c2ef195ab347a7b231c775c6eb042c 2024-11-16T05:59:07,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/86a13f9f77174a7db147fe70f584ec96 2024-11-16T05:59:07,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/e98aa508730946708aea6cbc60fb1c8a 2024-11-16T05:59:07,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c49b71d7f5f042d9bc44254563131a1c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c49b71d7f5f042d9bc44254563131a1c 2024-11-16T05:59:07,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/397631fd658440419671781f811aa0dd 2024-11-16T05:59:07,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c823374827c4458fbf4202d1eac89d08 2024-11-16T05:59:07,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c51dfed79b7e4f6b9f84512bba1d554d to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c51dfed79b7e4f6b9f84512bba1d554d 2024-11-16T05:59:07,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/8a458185ce644f85b63425b526b3610f 2024-11-16T05:59:07,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/c5b28c868633434ba138a1d7e37958b0 2024-11-16T05:59:07,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f8e17e5c46fa4e2d88183633ff6812e9 2024-11-16T05:59:07,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/08ce259b729f4dc4b1d42dbf1e5a3123 2024-11-16T05:59:07,165 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits/950.seqid, newMaxSeqId=950, maxSeqId=4 2024-11-16T05:59:07,166 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. 2024-11-16T05:59:07,166 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for a01be5863776074fa8e10401262fcc00: Waiting for close lock at 1731736745384Running coprocessor pre-close hooks at 1731736745384Disabling compacts and flushes for region at 1731736745384Disabling writes for close at 1731736745384Obtaining lock to block concurrent updates at 1731736745385 (+1 ms)Preparing flush snapshotting stores in a01be5863776074fa8e10401262fcc00 at 1731736745385Finished memstore snapshotting TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731736745385Flushing stores of TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00. at 1731736745386 (+1 ms)Flushing a01be5863776074fa8e10401262fcc00/A: creating writer at 1731736745386Flushing mob file a01be5863776074fa8e10401262fcc00/A: appending metadata at 1731736745391 (+5 ms)Flushing mob file a01be5863776074fa8e10401262fcc00/A: closing flushed file at 1731736745391Flushing a01be5863776074fa8e10401262fcc00/A: appending metadata at 1731736745807 (+416 ms)Flushing a01be5863776074fa8e10401262fcc00/A: closing flushed file at 1731736745807Flushing a01be5863776074fa8e10401262fcc00/B: creating writer at 1731736746219 (+412 ms)Flushing a01be5863776074fa8e10401262fcc00/B: appending metadata at 1731736746223 (+4 ms)Flushing a01be5863776074fa8e10401262fcc00/B: closing flushed file at 1731736746223Flushing a01be5863776074fa8e10401262fcc00/C: creating writer at 1731736746638 (+415 ms)Flushing a01be5863776074fa8e10401262fcc00/C: appending metadata at 1731736746640 (+2 ms)Flushing a01be5863776074fa8e10401262fcc00/C: closing flushed file at 1731736746640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@670fcd69: reopening flushed file at 1731736747050 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4935068b: reopening flushed file at 1731736747056 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11ea07ee: reopening flushed file at 1731736747062 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a01be5863776074fa8e10401262fcc00 in 1685ms, sequenceid=947, compaction requested=true at 1731736747070 (+8 ms)Writing region close event to WAL at 1731736747162 (+92 ms)Running coprocessor post-close hooks at 1731736747166 (+4 ms)Closed at 1731736747166 2024-11-16T05:59:07,168 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=a01be5863776074fa8e10401262fcc00, regionState=CLOSED 2024-11-16T05:59:07,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 because future has completed 2024-11-16T05:59:07,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T05:59:07,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure a01be5863776074fa8e10401262fcc00, server=3456ee6a3164,45735,1731736558702 in 1.9400 sec 2024-11-16T05:59:07,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-16T05:59:07,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a01be5863776074fa8e10401262fcc00, UNASSIGN in 1.9450 sec 2024-11-16T05:59:07,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T05:59:07,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9510 sec 2024-11-16T05:59:07,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736747181"}]},"ts":"1731736747181"} 2024-11-16T05:59:07,183 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T05:59:07,183 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T05:59:07,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9640 sec 2024-11-16T05:59:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T05:59:07,374 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:59:07,376 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T05:59:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,379 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:59:07,380 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C, FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits] 2024-11-16T05:59:07,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/245d0c8d570f447a8dabf9d5e27722d9 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/245d0c8d570f447a8dabf9d5e27722d9 2024-11-16T05:59:07,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/27d4b5ebb23c4760998b0dbf8139e334 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/27d4b5ebb23c4760998b0dbf8139e334 2024-11-16T05:59:07,397 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ee5b05cabed848448ef19b37efcd1c38 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/A/ee5b05cabed848448ef19b37efcd1c38 2024-11-16T05:59:07,401 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1a66fab5be8f47759829b7866e705f10 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1a66fab5be8f47759829b7866e705f10 2024-11-16T05:59:07,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1b709c2a36f6494cb60ff4ae6579aafa to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/1b709c2a36f6494cb60ff4ae6579aafa 2024-11-16T05:59:07,405 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/d8d82a6a5fb94803baffe95ab58e218c to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/B/d8d82a6a5fb94803baffe95ab58e218c 2024-11-16T05:59:07,409 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/26143c64bd5c4f35b32484796dafd1c3 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/26143c64bd5c4f35b32484796dafd1c3 2024-11-16T05:59:07,411 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/a89aac7e452a466585fbdcbfa9df75ff to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/a89aac7e452a466585fbdcbfa9df75ff 2024-11-16T05:59:07,413 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f6b91b2b7a484b2b8ad8cf436d2a0b1f to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/C/f6b91b2b7a484b2b8ad8cf436d2a0b1f 2024-11-16T05:59:07,418 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits/950.seqid to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00/recovered.edits/950.seqid 2024-11-16T05:59:07,419 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/default/TestAcidGuarantees/a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,419 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T05:59:07,420 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:59:07,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T05:59:07,427 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116092c0e624ddb40b7b107d2d04e62a2f0_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116092c0e624ddb40b7b107d2d04e62a2f0_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,429 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628c04426f1434a6b8c1c713248ea6adf_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628c04426f1434a6b8c1c713248ea6adf_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,432 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111629c0b450acc9484389e44d015fc989b2_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111629c0b450acc9484389e44d015fc989b2_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,434 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164130d49ce3e649d7997d8737e9c11da8_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164130d49ce3e649d7997d8737e9c11da8_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,436 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111641f4da57d39a4b34bb45c4582b7aa89b_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111641f4da57d39a4b34bb45c4582b7aa89b_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,438 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164396c29a933147c78b950eb4257791a8_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164396c29a933147c78b950eb4257791a8_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,441 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111671ea6dab8481441f80cf758bd9155df5_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111671ea6dab8481441f80cf758bd9155df5_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,443 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111672cba8b36f064e7b8a721e68c8d36d4e_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111672cba8b36f064e7b8a721e68c8d36d4e_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,445 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cc8e534b450434db890807bf6f92a5f_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cc8e534b450434db890807bf6f92a5f_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,447 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697b80ff5edd7481b8405a6a226cbbdbb_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697b80ff5edd7481b8405a6a226cbbdbb_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,450 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697c8a4ccd7e748c1aa2c2d3052e8c338_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697c8a4ccd7e748c1aa2c2d3052e8c338_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,453 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b0b08a248e674022a6b4abeebbdf5676_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b0b08a248e674022a6b4abeebbdf5676_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,455 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bfe4222559864dcfa4556d4e0a69c60f_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bfe4222559864dcfa4556d4e0a69c60f_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7c71b45aa3f469783bdc06fdef63aed_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7c71b45aa3f469783bdc06fdef63aed_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb11f5a340d2484382a6a76330125995_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cb11f5a340d2484382a6a76330125995_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e37751c4c05340aeaf8dee7d8290e1d7_a01be5863776074fa8e10401262fcc00 to hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e37751c4c05340aeaf8dee7d8290e1d7_a01be5863776074fa8e10401262fcc00 2024-11-16T05:59:07,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T05:59:07,466 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,469 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T05:59:07,472 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T05:59:07,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,474 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T05:59:07,474 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731736747474"}]},"ts":"9223372036854775807"} 2024-11-16T05:59:07,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T05:59:07,476 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a01be5863776074fa8e10401262fcc00, NAME => 'TestAcidGuarantees,,1731736712143.a01be5863776074fa8e10401262fcc00.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T05:59:07,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T05:59:07,477 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731736747476"}]},"ts":"9223372036854775807"} 2024-11-16T05:59:07,479 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T05:59:07,480 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T05:59:07,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-16T05:59:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40477 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T05:59:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T05:59:07,484 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T05:59:07,495 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 249), OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 509), ProcessCount=11 (was 11), AvailableMemoryMB=2988 (was 3389) 2024-11-16T05:59:07,495 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:59:07,496 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:59:07,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:59:07,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:07,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:07,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:59:07,496 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:59:07,496 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=261653369, stopped=false 2024-11-16T05:59:07,497 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,40477,1731736558105 2024-11-16T05:59:07,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:59:07,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:59:07,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:59:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:59:07,502 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:59:07,502 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:59:07,502 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:59:07,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:07,503 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,45735,1731736558702' ***** 2024-11-16T05:59:07,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:59:07,503 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:59:07,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:59:07,503 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:59:07,503 INFO [RS:0;3456ee6a3164:45735 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:59:07,503 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,45735,1731736558702 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:45735. 2024-11-16T05:59:07,504 DEBUG [RS:0;3456ee6a3164:45735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:59:07,504 DEBUG [RS:0;3456ee6a3164:45735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:59:07,504 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:59:07,505 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T05:59:07,505 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T05:59:07,505 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:07,505 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:59:07,505 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:59:07,505 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:59:07,505 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:59:07,505 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:59:07,506 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T05:59:07,506 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:59:07,506 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:59:07,529 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/info/a13bc9a3dfd942e491cefb34dc95bb19 is 95, key is TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969./info:/1731736598422/DeleteFamily/seqid=0 2024-11-16T05:59:07,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742480_1656 (size=6514) 2024-11-16T05:59:07,574 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:59:07,706 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:07,906 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:07,934 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/info/a13bc9a3dfd942e491cefb34dc95bb19 2024-11-16T05:59:07,956 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/ns/93318ae78909482286ba312bffc2b5a9 is 93, key is TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969./ns:/1731736598379/DeleteFamily/seqid=0 2024-11-16T05:59:07,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742481_1657 (size=5906) 2024-11-16T05:59:08,106 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:08,307 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:08,361 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/ns/93318ae78909482286ba312bffc2b5a9 2024-11-16T05:59:08,389 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/rep_barrier/16b1b765b99b462bb4d6d057cbe199a4 is 102, key is TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969./rep_barrier:/1731736598379/DeleteFamily/seqid=0 2024-11-16T05:59:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742482_1658 (size=6025) 2024-11-16T05:59:08,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T05:59:08,507 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:08,707 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:08,793 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/rep_barrier/16b1b765b99b462bb4d6d057cbe199a4 2024-11-16T05:59:08,818 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/table/6327504140a04726ad6c91e12af3db55 is 96, key is TestAcidGuarantees,,1731736562906.d3e0754fed086c5654deac27f4f83969./table:/1731736598379/DeleteFamily/seqid=0 2024-11-16T05:59:08,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742483_1659 (size=5936) 2024-11-16T05:59:08,907 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:09,108 DEBUG [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:59:09,222 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/table/6327504140a04726ad6c91e12af3db55 2024-11-16T05:59:09,231 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/info/a13bc9a3dfd942e491cefb34dc95bb19 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/info/a13bc9a3dfd942e491cefb34dc95bb19 2024-11-16T05:59:09,237 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/info/a13bc9a3dfd942e491cefb34dc95bb19, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T05:59:09,238 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/ns/93318ae78909482286ba312bffc2b5a9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/ns/93318ae78909482286ba312bffc2b5a9 2024-11-16T05:59:09,241 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/ns/93318ae78909482286ba312bffc2b5a9, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T05:59:09,242 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/rep_barrier/16b1b765b99b462bb4d6d057cbe199a4 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/rep_barrier/16b1b765b99b462bb4d6d057cbe199a4 2024-11-16T05:59:09,245 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/rep_barrier/16b1b765b99b462bb4d6d057cbe199a4, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T05:59:09,245 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/.tmp/table/6327504140a04726ad6c91e12af3db55 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/table/6327504140a04726ad6c91e12af3db55 2024-11-16T05:59:09,249 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/table/6327504140a04726ad6c91e12af3db55, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T05:59:09,249 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false 2024-11-16T05:59:09,254 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T05:59:09,254 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:59:09,255 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:59:09,255 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736747505Running coprocessor pre-close hooks at 1731736747505Disabling compacts and flushes for region at 1731736747505Disabling writes for close at 1731736747505Obtaining lock to block concurrent updates at 1731736747506 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731736747506Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731736747506Flushing stores of hbase:meta,,1.1588230740 at 1731736747507 (+1 ms)Flushing 1588230740/info: creating writer at 1731736747507Flushing 1588230740/info: appending metadata at 1731736747529 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731736747529Flushing 1588230740/ns: creating writer at 1731736747943 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731736747955 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731736747956 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731736748369 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731736748389 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731736748389Flushing 1588230740/table: creating writer at 1731736748800 (+411 ms)Flushing 1588230740/table: appending metadata at 1731736748817 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731736748817Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b83363: reopening flushed file at 1731736749230 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51097a52: reopening flushed file at 1731736749237 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43291332: reopening flushed file at 1731736749241 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76fd4cba: reopening flushed file at 1731736749245 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false at 1731736749249 (+4 ms)Writing region close event to WAL at 1731736749251 (+2 ms)Running coprocessor post-close hooks at 1731736749254 (+3 ms)Closed at 1731736749255 (+1 ms) 2024-11-16T05:59:09,255 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:59:09,308 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,45735,1731736558702; all regions closed. 2024-11-16T05:59:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741834_1010 (size=25344) 2024-11-16T05:59:09,319 DEBUG [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/oldWALs 2024-11-16T05:59:09,319 INFO [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C45735%2C1731736558702.meta:.meta(num 1731736562534) 2024-11-16T05:59:09,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741832_1008 (size=21301923) 2024-11-16T05:59:09,322 DEBUG [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/oldWALs 2024-11-16T05:59:09,322 INFO [RS:0;3456ee6a3164:45735 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C45735%2C1731736558702:(num 1731736561625) 2024-11-16T05:59:09,322 DEBUG [RS:0;3456ee6a3164:45735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:59:09,322 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:59:09,323 INFO [RS:0;3456ee6a3164:45735 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:59:09,323 INFO [RS:0;3456ee6a3164:45735 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T05:59:09,323 INFO [RS:0;3456ee6a3164:45735 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:59:09,323 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:59:09,324 INFO [RS:0;3456ee6a3164:45735 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45735 2024-11-16T05:59:09,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,45735,1731736558702 2024-11-16T05:59:09,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:59:09,327 INFO [RS:0;3456ee6a3164:45735 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:59:09,327 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,45735,1731736558702] 2024-11-16T05:59:09,329 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,45735,1731736558702 already deleted, retry=false 2024-11-16T05:59:09,329 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,45735,1731736558702 expired; onlineServers=0 2024-11-16T05:59:09,329 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,40477,1731736558105' ***** 2024-11-16T05:59:09,329 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:59:09,329 INFO [M:0;3456ee6a3164:40477 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:59:09,329 INFO [M:0;3456ee6a3164:40477 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:59:09,329 DEBUG [M:0;3456ee6a3164:40477 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:59:09,329 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:59:09,329 DEBUG [M:0;3456ee6a3164:40477 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:59:09,329 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736561267 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736561267,5,FailOnTimeoutGroup] 2024-11-16T05:59:09,329 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736561264 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736561264,5,FailOnTimeoutGroup] 2024-11-16T05:59:09,330 INFO [M:0;3456ee6a3164:40477 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:59:09,330 INFO [M:0;3456ee6a3164:40477 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:59:09,330 DEBUG [M:0;3456ee6a3164:40477 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:59:09,330 INFO [M:0;3456ee6a3164:40477 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:59:09,330 INFO [M:0;3456ee6a3164:40477 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:59:09,330 ERROR [M:0;3456ee6a3164:40477 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T05:59:09,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:59:09,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:59:09,331 INFO [M:0;3456ee6a3164:40477 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:59:09,331 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:59:09,331 DEBUG [M:0;3456ee6a3164:40477 {}] zookeeper.ZKUtil(347): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T05:59:09,331 WARN [M:0;3456ee6a3164:40477 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T05:59:09,332 INFO [M:0;3456ee6a3164:40477 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/.lastflushedseqids 2024-11-16T05:59:09,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742484_1660 (size=72) 2024-11-16T05:59:09,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:59:09,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45735-0x100471a33620001, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:59:09,430 INFO [RS:0;3456ee6a3164:45735 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:59:09,431 INFO [RS:0;3456ee6a3164:45735 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,45735,1731736558702; zookeeper connection closed. 2024-11-16T05:59:09,431 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a733d0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a733d0c 2024-11-16T05:59:09,432 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T05:59:09,741 INFO [M:0;3456ee6a3164:40477 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:59:09,742 INFO [M:0;3456ee6a3164:40477 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:59:09,742 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:59:09,742 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:59:09,742 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:59:09,742 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:59:09,742 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:59:09,742 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=761.69 KB heapSize=932.54 KB 2024-11-16T05:59:09,760 DEBUG [M:0;3456ee6a3164:40477 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0341fb27447e4278a64fa839dae40fc9 is 82, key is hbase:meta,,1/info:regioninfo/1731736562616/Put/seqid=0 2024-11-16T05:59:09,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742485_1661 (size=5672) 2024-11-16T05:59:09,763 INFO [M:0;3456ee6a3164:40477 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0341fb27447e4278a64fa839dae40fc9 2024-11-16T05:59:09,782 DEBUG [M:0;3456ee6a3164:40477 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f4ee89319ca40998af06a01c358becb is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1731736715230/Put/seqid=0 2024-11-16T05:59:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742486_1662 (size=42975) 2024-11-16T05:59:10,187 INFO [M:0;3456ee6a3164:40477 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=761.14 KB at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f4ee89319ca40998af06a01c358becb 2024-11-16T05:59:10,195 INFO [M:0;3456ee6a3164:40477 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f4ee89319ca40998af06a01c358becb 2024-11-16T05:59:10,212 DEBUG [M:0;3456ee6a3164:40477 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d061a8520f3f494a90961ab2470efa83 is 69, key is 3456ee6a3164,45735,1731736558702/rs:state/1731736561402/Put/seqid=0 2024-11-16T05:59:10,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073742487_1663 (size=5156) 2024-11-16T05:59:10,616 INFO [M:0;3456ee6a3164:40477 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d061a8520f3f494a90961ab2470efa83 2024-11-16T05:59:10,625 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0341fb27447e4278a64fa839dae40fc9 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0341fb27447e4278a64fa839dae40fc9 2024-11-16T05:59:10,632 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0341fb27447e4278a64fa839dae40fc9, entries=8, sequenceid=2117, filesize=5.5 K 2024-11-16T05:59:10,633 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f4ee89319ca40998af06a01c358becb as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f4ee89319ca40998af06a01c358becb 2024-11-16T05:59:10,637 INFO [M:0;3456ee6a3164:40477 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f4ee89319ca40998af06a01c358becb 2024-11-16T05:59:10,637 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f4ee89319ca40998af06a01c358becb, entries=156, sequenceid=2117, filesize=42.0 K 2024-11-16T05:59:10,638 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d061a8520f3f494a90961ab2470efa83 as hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d061a8520f3f494a90961ab2470efa83 2024-11-16T05:59:10,642 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45913/user/jenkins/test-data/7b26af77-91ef-0ca4-901c-f30099ae3e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d061a8520f3f494a90961ab2470efa83, entries=1, sequenceid=2117, filesize=5.0 K 2024-11-16T05:59:10,642 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(3140): Finished flush of dataSize ~761.69 KB/779972, heapSize ~932.24 KB/954616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 900ms, sequenceid=2117, compaction requested=false 2024-11-16T05:59:10,646 INFO [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:59:10,647 DEBUG [M:0;3456ee6a3164:40477 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736749742Disabling compacts and flushes for region at 1731736749742Disabling writes for close at 1731736749742Obtaining lock to block concurrent updates at 1731736749742Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731736749742Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779972, getHeapSize=954856, getOffHeapSize=0, getCellsCount=2298 at 1731736749742Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731736749743 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731736749743Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731736749759 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731736749759Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731736749766 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731736749782 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731736749782Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731736750196 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731736750212 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731736750212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed3daf7: reopening flushed file at 1731736750624 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59f0143c: reopening flushed file at 1731736750632 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54019f3e: reopening flushed file at 1731736750637 (+5 ms)Finished flush of dataSize ~761.69 KB/779972, heapSize ~932.24 KB/954616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 900ms, sequenceid=2117, compaction requested=false at 1731736750642 (+5 ms)Writing region close event to WAL at 1731736750646 (+4 ms)Closed at 1731736750646 2024-11-16T05:59:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41419 is added to blk_1073741830_1006 (size=918720) 2024-11-16T05:59:10,650 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:59:10,650 INFO [M:0;3456ee6a3164:40477 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T05:59:10,650 INFO [M:0;3456ee6a3164:40477 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40477 2024-11-16T05:59:10,651 INFO [M:0;3456ee6a3164:40477 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:59:10,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:59:10,754 INFO [M:0;3456ee6a3164:40477 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:59:10,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40477-0x100471a33620000, quorum=127.0.0.1:59104, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:59:10,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:59:10,767 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:59:10,767 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:59:10,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:59:10,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.log.dir/,STOPPED} 2024-11-16T05:59:10,771 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T05:59:10,771 WARN [BP-1783175189-172.17.0.2-1731736555553 heartbeating to localhost/127.0.0.1:45913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T05:59:10,771 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T05:59:10,771 WARN [BP-1783175189-172.17.0.2-1731736555553 heartbeating to localhost/127.0.0.1:45913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1783175189-172.17.0.2-1731736555553 (Datanode Uuid b3959656-3ba6-40ef-9099-78748206e8cb) service to localhost/127.0.0.1:45913 2024-11-16T05:59:10,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data1/current/BP-1783175189-172.17.0.2-1731736555553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:59:10,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/cluster_525bdd9a-15d4-2101-d7b4-7885866e8663/data/data2/current/BP-1783175189-172.17.0.2-1731736555553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:59:10,776 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T05:59:10,783 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:59:10,783 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:59:10,783 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:59:10,783 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:59:10,784 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/aaa36547-41d7-ec25-b392-4c2ad8f4942e/hadoop.log.dir/,STOPPED} 2024-11-16T05:59:10,800 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T05:59:10,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down